fluentd222 1.16.2-x86_64-linux

data/lib/fluent/compat/call_super_mixin.rb

@@ -0,0 +1,76 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Compat
+    module CallSuperMixin
+      # This mixin is to prepend to 3rd party plugins of v0.12 APIs.
+      # In past, there were not strong rule to call super in #start, #before_shutdown and #shutdown.
+      # But v0.14 API requires to call super in these methods to setup/teardown plugin helpers and others.
+      # This mixin prepends method calls to call super forcedly if checker returns false (it shows Fluent::Plugin::Base#methods wasn't called)
+
+      def self.prepended(klass)
+        @@_super_start ||= {}
+        @@_super_before_shutdown ||= {}
+        @@_super_shutdown ||= {}
+
+        # ancestors[0]: this module
+        # ancestors[1]: prepended class (plugin itself)
+        method_search = ->(ancestors, method){
+          closest = ancestors[2, ancestors.size - 2].index{|m| m.method_defined?(method) }
+          ancestors[2 + closest].instance_method(method)
+        }
+        @@_super_start[klass]           = method_search.call(klass.ancestors, :start) # this returns Fluent::Compat::*#start (or helpers on it)
+        @@_super_before_shutdown[klass] = method_search.call(klass.ancestors, :before_shutdown)
+        @@_super_shutdown[klass]        = method_search.call(klass.ancestors, :shutdown)
+      end
+
+      def start
+        super
+        unless self.started?
+          @@_super_start[self.class].bind(self).call
+          # #super will reset logdev (especially in test), so this warn should be after calling it
+          log.warn "super was not called in #start: called it forcedly", plugin: self.class
+        end
+      end
+
+      def before_shutdown
+        super
+        unless self.before_shutdown?
+          log.warn "super was not called in #before_shutdown: calling it forcedly", plugin: self.class
+          @@_super_before_shutdown[self.class].bind(self).call
+        end
+      end
+
+      def stop
+        klass = self.class
+        @@_super_start.delete(klass)
+        @@_super_before_shutdown.delete(klass)
+        @@_super_shutdown.delete(klass)
+
+        super
+      end
+
+      def shutdown
+        super
+        unless self.shutdown?
+          log.warn "super was not called in #shutdown: calling it forcedly", plugin: self.class
+          @@_super_shutdown[self.class].bind(self).call
+        end
+      end
+    end
+  end
+end

data/lib/fluent/compat/detach_process_mixin.rb

@@ -0,0 +1,33 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Compat
+    module DetachProcessMixin
+      def detach_process
+        log.warn "#{__method__} is not supported in this version. ignored."
+        yield
+      end
+    end
+
+    module DetachMultiProcessMixin
+      def detach_multi_process
+        log.warn "#{__method__} is not supported in this version. ignored."
+        yield
+      end
+    end
+  end
+end

data/lib/fluent/compat/exec_util.rb

@@ -0,0 +1,129 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'msgpack'
+require 'yajl'
+
+require 'fluent/engine'
+require 'fluent/plugin'
+require 'fluent/parser'
+
+module Fluent
+  module Compat
+    module ExecUtil
+      SUPPORTED_FORMAT = {
+        'tsv' => :tsv,
+        'json' => :json,
+        'msgpack' => :msgpack,
+      }
+
+      class Parser
+        def initialize(on_message)
+          @on_message = on_message
+        end
+      end
+
+      class TextParserWrapperParser < Parser
+        def initialize(conf, on_message)
+          @parser = Plugin.new_parser(conf['format'])
+          @parser.configure(conf)
+          super(on_message)
+        end
+
+        def call(io)
+          io.each_line(&method(:each_line))
+        end
+
+        def each_line(line)
+          line.chomp!
+          @parser.parse(line) { |time, record|
+            @on_message.call(record, time)
+          }
+        end
+      end
+
+      class TSVParser < Parser
+        def initialize(keys, on_message)
+          @keys = keys
+          super(on_message)
+        end
+
+        def call(io)
+          io.each_line(&method(:each_line))
+        end
+
+        def each_line(line)
+          line.chomp!
+          vals = line.split("\t")
+
+          record = Hash[@keys.zip(vals)]
+
+          @on_message.call(record)
+        end
+      end
+
+      class JSONParser < Parser
+        def call(io)
+          y = Yajl::Parser.new
+          y.on_parse_complete = @on_message
+          y.parse(io)
+        end
+      end
+
+      class MessagePackParser < Parser
+        def call(io)
+          @u = Fluent::MessagePackFactory.msgpack_unpacker(io)
+          begin
+            @u.each(&@on_message)
+          rescue EOFError
+          end
+        end
+      end
+
+      class Formatter
+      end
+
+      class TSVFormatter < Formatter
+        def initialize(in_keys)
+          @in_keys = in_keys
+          super()
+        end
+
+        def call(record, out)
+          last = @in_keys.length-1
+          for i in 0..last
+            key = @in_keys[i]
+            out << record[key].to_s
+            out << "\t" if i != last
+          end
+          out << "\n"
+        end
+      end
+
+      class JSONFormatter < Formatter
+        def call(record, out)
+          out << Yajl.dump(record) << "\n"
+        end
+      end
+
+      class MessagePackFormatter < Formatter
+        def call(record, out)
+          record.to_msgpack(out)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/compat/file_util.rb

@@ -0,0 +1,54 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Compat
+    module FileUtil
+      # Check file is writable if file exists
+      # Check directory is writable if file does not exist
+      #
+      # @param [String] path File path
+      # @return [Boolean] file is writable or not
+      def writable?(path)
+        return false if File.directory?(path)
+        return File.writable?(path) if File.exist?(path)
+
+        dirname = File.dirname(path)
+        return false if !File.directory?(dirname)
+        File.writable?(dirname)
+      end
+      module_function :writable?
+
+      # Check file is writable in conjunction with mkdir_p(dirname(path))
+      #
+      # @param [String] path File path
+      # @return [Boolean] file writable or not
+      def writable_p?(path)
+        return false if File.directory?(path)
+        return File.writable?(path) if File.exist?(path)
+
+        dirname = File.dirname(path)
+        until File.exist?(dirname)
+          dirname = File.dirname(dirname)
+        end
+
+        return false if !File.directory?(dirname)
+        File.writable?(dirname)
+      end
+      module_function :writable_p?
+    end
+  end
+end

data/lib/fluent/compat/filter.rb

@@ -0,0 +1,68 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/filter'
+require 'fluent/compat/call_super_mixin'
+require 'fluent/compat/formatter_utils'
+require 'fluent/compat/parser_utils'
+
+module Fluent
+  module Compat
+    class Filter < Fluent::Plugin::Filter
+      # TODO: warn when deprecated
+
+      helpers_internal :inject
+
+      def initialize
+        super
+        unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
+          self.class.prepend Fluent::Compat::CallSuperMixin
+        end
+      end
+
+      def configure(conf)
+        ParserUtils.convert_parser_conf(conf)
+        FormatterUtils.convert_formatter_conf(conf)
+
+        super
+      end
+
+      # These definitions are to get instance methods of superclass of 3rd party plugins
+      # to make it sure to call super
+      def start
+        super
+
+        if instance_variable_defined?(:@formatter) && @inject_config
+          unless @formatter.class.ancestors.include?(Fluent::Compat::HandleTagAndTimeMixin)
+            if @formatter.respond_to?(:owner) && !@formatter.owner
+              @formatter.owner = self
+              @formatter.singleton_class.prepend FormatterUtils::InjectMixin
+            end
+          end
+        end
+      end
+
+      def before_shutdown
+        super
+      end
+
+      def shutdown
+        super
+      end
+    end
+  end
+end

data/lib/fluent/compat/formatter.rb

@@ -0,0 +1,111 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/formatter'
+require 'fluent/compat/handle_tag_and_time_mixin'
+require 'fluent/compat/structured_format_mixin'
+
+require 'fluent/plugin/formatter_out_file'
+require 'fluent/plugin/formatter_stdout'
+require 'fluent/plugin/formatter_json'
+require 'fluent/plugin/formatter_hash'
+require 'fluent/plugin/formatter_msgpack'
+require 'fluent/plugin/formatter_ltsv'
+require 'fluent/plugin/formatter_csv'
+require 'fluent/plugin/formatter_single_value'
+
+module Fluent
+  module Compat
+    class Formatter < Fluent::Plugin::Formatter
+      # TODO: warn when deprecated
+    end
+
+    module TextFormatter
+      def self.register_template(type, template)
+        # TODO: warn when deprecated to use Plugin.register_formatter directly
+        if template.is_a?(Class)
+          Fluent::Plugin.register_formatter(type, template)
+        elsif template.respond_to?(:call) && template.arity == 3 # Proc.new { |tag, time, record| }
+          Fluent::Plugin.register_formatter(type, Proc.new { ProcWrappedFormatter.new(template) })
+        elsif template.respond_to?(:call)
+          Fluent::Plugin.register_formatter(type, template)
+        else
+          raise ArgumentError, "Template for formatter must be a Class or callable object"
+        end
+      end
+
+      def self.lookup(type)
+        # TODO: warn when deprecated to use Plugin.new_formatter(type, parent: plugin)
+        Fluent::Plugin.new_formatter(type)
+      end
+
+      # Keep backward-compatibility
+      def self.create(conf)
+        # TODO: warn when deprecated
+        format = conf['format']
+        if format.nil?
+          raise ConfigError, "'format' parameter is required"
+        end
+
+        formatter = lookup(format)
+        if formatter.respond_to?(:configure)
+          formatter.configure(conf)
+        end
+        formatter
+      end
+
+      HandleTagAndTimeMixin = Fluent::Compat::HandleTagAndTimeMixin
+      StructuredFormatMixin = Fluent::Compat::StructuredFormatMixin
+
+      class ProcWrappedFormatter < Fluent::Plugin::ProcWrappedFormatter
+        # TODO: warn when deprecated
+      end
+
+      class OutFileFormatter < Fluent::Plugin::OutFileFormatter
+        # TODO: warn when deprecated
+      end
+
+      class StdoutFormatter < Fluent::Plugin::StdoutFormatter
+        # TODO: warn when deprecated
+      end
+
+      class JSONFormatter < Fluent::Plugin::JSONFormatter
+        # TODO: warn when deprecated
+      end
+
+      class HashFormatter < Fluent::Plugin::HashFormatter
+        # TODO: warn when deprecated
+      end
+
+      class MessagePackFormatter < Fluent::Plugin::MessagePackFormatter
+        # TODO: warn when deprecated
+      end
+
+      class LabeledTSVFormatter < Fluent::Plugin::LabeledTSVFormatter
+        # TODO: warn when deprecated
+      end
+
+      class CsvFormatter < Fluent::Plugin::CsvFormatter
+        # TODO: warn when deprecated
+      end
+
+      class SingleValueFormatter < Fluent::Plugin::SingleValueFormatter
+        # TODO: warn when deprecated
+      end
+    end
+  end
+end

data/lib/fluent/compat/formatter_utils.rb

@@ -0,0 +1,85 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/compat_parameters'
+
+module Fluent
+  module Compat
+    module FormatterUtils
+      INJECT_PARAMS = Fluent::PluginHelper::CompatParameters::INJECT_PARAMS
+      FORMATTER_PARAMS = Fluent::PluginHelper::CompatParameters::FORMATTER_PARAMS
+
+      module InjectMixin
+        def format(tag, time, record)
+          r = owner.inject_values_to_record(tag, time, record)
+          super(tag, time, r)
+        end
+      end
+
+      def self.convert_formatter_conf(conf)
+        return if conf.elements(name: 'inject').first || conf.elements(name: 'format').first
+
+        inject_params = {}
+        INJECT_PARAMS.each do |older, newer|
+          next unless newer
+          if conf.has_key?(older)
+            inject_params[newer] = conf[older]
+          end
+        end
+
+        if conf.has_key?('include_time_key') && Fluent::Config.bool_value(conf['include_time_key'])
+          inject_params['time_key'] ||= 'time'
+          inject_params['time_type'] ||= 'string'
+        end
+        if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
+          inject_params['time_type'] = 'unixtime'
+        end
+        if conf.has_key?('localtime') || conf.has_key?('utc')
+          if conf.has_key?('localtime') && conf.has_key?('utc')
+            raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
+          elsif conf.has_key?('localtime')
+            inject_params['localtime'] = Fluent::Config.bool_value(conf['localtime'])
+          elsif conf.has_key?('utc')
+            inject_params['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
+            # Specifying "localtime false" means using UTC in TimeFormatter
+            # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
+            # There are difference between "Z" and "+0000" in timezone formatting.
+            # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
+          end
+        end
+
+        if conf.has_key?('include_tag_key') && Fluent::Config.bool_value(conf['include_tag_key'])
+          inject_params['tag_key'] ||= 'tag'
+        end
+
+        unless inject_params.empty?
+          conf.elements << Fluent::Config::Element.new('inject', '', inject_params, [])
+        end
+
+        formatter_params = {}
+        FORMATTER_PARAMS.each do |older, newer|
+          next unless newer
+          if conf.has_key?(older)
+            formatter_params[newer] = conf[older]
+          end
+        end
+        unless formatter_params.empty?
+          conf.elements << Fluent::Config::Element.new('format', '', formatter_params, [])
+        end
+      end
+    end
+  end
+end

data/lib/fluent/compat/handle_tag_and_time_mixin.rb

@@ -0,0 +1,62 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/time' # TimeFormatter
+
+module Fluent
+  module Compat
+    module HandleTagAndTimeMixin
+      def self.included(klass)
+        klass.instance_eval {
+          config_param :include_time_key, :bool, default: false
+          config_param :time_key, :string, default: 'time'
+          config_param :time_format, :string, default: nil
+          config_param :time_as_epoch, :bool, default: false
+          config_param :include_tag_key, :bool, default: false
+          config_param :tag_key, :string, default: 'tag'
+          config_param :localtime, :bool, default: true
+          config_param :timezone, :string, default: nil
+        }
+      end
+
+      def configure(conf)
+        super
+
+        if conf['utc']
+          @localtime = false
+        end
+        @timef = Fluent::TimeFormatter.new(@time_format, @localtime, @timezone)
+        if @time_as_epoch && !@include_time_key
+          log.warn "time_as_epoch will be ignored because include_time_key is false"
+        end
+      end
+
+      def filter_record(tag, time, record)
+        if @include_tag_key
+          record[@tag_key] = tag
+        end
+        if @include_time_key
+          if @time_as_epoch
+            record[@time_key] = time.to_i
+          else
+            record[@time_key] = @timef.format(time)
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/fluent/compat/handle_tag_name_mixin.rb

@@ -0,0 +1,53 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/record_filter_mixin'
+
+module Fluent
+  module Compat
+    module HandleTagNameMixin
+      include RecordFilterMixin
+
+      attr_accessor :remove_tag_prefix, :remove_tag_suffix, :add_tag_prefix, :add_tag_suffix
+      def configure(conf)
+        super
+
+        @remove_tag_prefix = if conf.has_key?('remove_tag_prefix')
+                               Regexp.new('^' + Regexp.escape(conf['remove_tag_prefix']))
+                             else
+                               nil
+                             end
+
+        @remove_tag_suffix = if conf.has_key?('remove_tag_suffix')
+                               Regexp.new(Regexp.escape(conf['remove_tag_suffix']) + '$')
+                             else
+                               nil
+                             end
+
+        @add_tag_prefix = conf['add_tag_prefix']
+        @add_tag_suffix = conf['add_tag_suffix']
+      end
+
+      def filter_record(tag, time, record)
+        tag.sub!(@remove_tag_prefix, '') if @remove_tag_prefix
+        tag.sub!(@remove_tag_suffix, '') if @remove_tag_suffix
+        tag.insert(0, @add_tag_prefix) if @add_tag_prefix
+        tag << @add_tag_suffix if @add_tag_suffix
+        super(tag, time, record)
+      end
+    end
+  end
+end