fluentd 1.14.4-x64-mingw-ucrt

data/lib/fluent/match.rb

@@ -0,0 +1,187 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  class MatchPattern
+    def self.create(str)
+      if str == '**'
+        AllMatchPattern.new
+      else
+        GlobMatchPattern.new(str)
+      end
+    end
+  end
+
+  class AllMatchPattern < MatchPattern
+    def match(str)
+      true
+    end
+  end
+
+  class GlobMatchPattern < MatchPattern
+    def initialize(pat)
+      if pat.start_with?('/')
+        if pat.end_with?('/')
+          @regex = Regexp.new("\\A"+pat[1..-2]+"\\Z")
+          return
+        else
+          raise Fluent::ConfigError,  "invalid match - regex"
+        end
+      end
+
+      stack = []
+      regex = ['']
+      escape = false
+      dot = false
+
+      i = 0
+      while i < pat.length
+        c = pat[i,1]
+
+        if escape
+          regex.last << Regexp.escape(c)
+          escape = false
+          i += 1
+          next
+
+        elsif pat[i,2] == "**"
+          # recursive any
+          if dot
+            regex.last << "(?![^\\.])"
+            dot = false
+          end
+          if pat[i+2,1] == "."
+            regex.last << "(?:.*\\.|\\A)"
+            i += 3
+          else
+            regex.last << ".*"
+            i += 2
+          end
+          next
+
+        elsif dot
+          regex.last << "\\."
+          dot = false
+        end
+
+        if c == "\\"
+          escape = true
+
+        elsif c == "."
+          dot = true
+
+        elsif c == "*"
+          # any
+          regex.last << "[^\\.]*"
+
+          # TODO
+          #elsif c == "["
+          #  # character class
+          #  chars = ''
+          #  while i < pat.length
+          #    c = pat[i,1]
+          #    if c == "]"
+          #      break
+          #    else
+          #      chars << c
+          #    end
+          #    i += 1
+          #  end
+          #  regex.last << '['+Regexp.escape(chars).gsub("\\-",'-')+']'
+
+        elsif c == "{"
+          # or
+          stack.push []
+          regex.push ''
+
+        elsif c == "}" && !stack.empty?
+          stack.last << regex.pop
+          regex.last << Regexp.union(*stack.pop.map {|r| Regexp.new(r) }).to_s
+
+        elsif c == "," && !stack.empty?
+          stack.last << regex.pop
+          regex.push ''
+
+        elsif c =~ /[a-zA-Z0-9_]/
+          regex.last << c
+
+        else
+          regex.last << "\\#{c}"
+        end
+
+        i += 1
+      end
+
+      until stack.empty?
+        stack.last << regex.pop
+        regex.last << Regexp.union(*stack.pop).to_s
+      end
+
+      @regex = Regexp.new("\\A"+regex.last+"\\Z")
+    end
+
+    def match(str)
+      @regex.match?(str)
+    end
+  end
+
+  class OrMatchPattern < MatchPattern
+    def initialize(patterns)
+      @patterns = patterns
+    end
+
+    def match(str)
+      @patterns.any? {|pattern| pattern.match(str) }
+    end
+  end
+
+  class NoMatchMatch
+    def initialize(log)
+      @log = log
+      @count = 0
+      @warn_not_matched = true
+    end
+
+    def suppress_missing_match!
+      # for <label @FLUENT_LOG>
+      @warn_not_matched = false
+    end
+
+    def emit_events(tag, es)
+      return unless @warn_not_matched
+      # TODO use time instead of num of records
+      c = (@count += 1)
+      if c < 512
+        if Math.log(c) / Math.log(2) % 1.0 == 0
+          @log.warn "no patterns matched", tag: tag
+          return
+        end
+      else
+        if c % 512 == 0
+          @log.warn "no patterns matched", tag: tag
+          return
+        end
+      end
+      @log.on_trace { @log.trace "no patterns matched", tag: tag }
+    end
+
+    def start
+    end
+
+    def shutdown
+    end
+  end
+end

data/lib/fluent/mixin.rb

@@ -0,0 +1,31 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/record_filter_mixin'
+require 'fluent/compat/handle_tag_name_mixin'
+require 'fluent/compat/set_time_key_mixin'
+require 'fluent/compat/set_tag_key_mixin'
+require 'fluent/compat/type_converter'
+
+require 'fluent/time' # Fluent::TimeFormatter
+
+module Fluent
+  RecordFilterMixin = Fluent::Compat::RecordFilterMixin
+  HandleTagNameMixin = Fluent::Compat::HandleTagNameMixin
+  SetTimeKeyMixin = Fluent::Compat::SetTimeKeyMixin
+  SetTagKeyMixin = Fluent::Compat::SetTagKeyMixin
+  TypeConverter = Fluent::Compat::TypeConverter
+end

data/lib/fluent/msgpack_factory.rb

@@ -0,0 +1,106 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'msgpack'
+require 'fluent/time'
+
+module Fluent
+  module MessagePackFactory
+    @@engine_factory = nil
+
+    module Mixin
+      def msgpack_factory
+        unless @deprecated_log_done
+          deprecated_log('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.engine_factory')
+        end
+        MessagePackFactory.engine_factory
+      end
+
+      def msgpack_packer(*args)
+        unless @deprecated_log_done
+          deprecated_log('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.msgpack_packer')
+        end
+        MessagePackFactory.msgpack_packer(*args)
+      end
+
+      def msgpack_unpacker(*args)
+        unless @deprecated_log_done
+          deprecated_log('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.msgpack_unpacker')
+        end
+        MessagePackFactory.msgpack_unpacker(*args)
+      end
+
+      def deprecated_log(str)
+        if $log
+          $log.warn(str)
+          @deprecated_log_done = true
+        end
+      end
+    end
+
+    def self.engine_factory(enable_time_support: false)
+      @@engine_factory || factory(enable_time_support: enable_time_support)
+    end
+
+    def self.msgpack_packer(*args)
+      engine_factory.packer(*args)
+    end
+
+    def self.msgpack_unpacker(*args)
+      engine_factory.unpacker(*args)
+    end
+
+    def self.factory(enable_time_support: false)
+      factory = MessagePack::Factory.new
+      factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)
+      if enable_time_support
+        factory.register_type(
+          MessagePack::Timestamp::TYPE, Time,
+          packer: MessagePack::Time::Packer,
+          unpacker: MessagePack::Time::Unpacker)
+      end
+      factory
+    end
+
+    def self.packer(*args)
+      factory.packer(*args)
+    end
+
+    def self.unpacker(*args)
+      factory.unpacker(*args)
+    end
+
+    def self.init(enable_time_support: false)
+      factory = MessagePack::Factory.new
+      factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)
+      if enable_time_support
+        factory.register_type(
+          MessagePack::Timestamp::TYPE, Time,
+          packer: MessagePack::Time::Packer,
+          unpacker: MessagePack::Time::Unpacker)
+      end
+      @@engine_factory = factory
+    end
+
+    def self.thread_local_msgpack_packer
+      Thread.current[:local_msgpack_packer] ||= MessagePackFactory.engine_factory.packer
+    end
+
+    def self.thread_local_msgpack_unpacker
+      Thread.current[:local_msgpack_unpacker] ||= MessagePackFactory.engine_factory.unpacker
+    end
+  end
+end

data/lib/fluent/oj_options.rb

@@ -0,0 +1,62 @@
+require 'fluent/config/types'
+
+module Fluent
+  class OjOptions
+    OPTIONS = {
+      'bigdecimal_load': :symbol,
+      'max_nesting': :integer,
+      'mode': :symbol,
+      'use_to_json': :bool
+    }
+
+    ALLOWED_VALUES = {
+      'bigdecimal_load': %i[bigdecimal float auto],
+      'mode': %i[strict null compat json rails object custom]
+    }
+
+    DEFAULTS = {
+      'bigdecimal_load': :float,
+      'mode': :compat,
+      'use_to_json': true
+    }
+
+    @@available = false
+
+    def self.available?
+      @@available
+    end
+
+    def self.load_env
+      options = self.get_options
+      begin
+        require 'oj'
+        Oj.default_options = options
+        @@available = true
+      rescue LoadError
+        @@available = false
+      end
+      options
+    end
+
+    private
+
+    def self.get_options
+      options = {}
+      DEFAULTS.each { |key, value| options[key] = value }
+
+      OPTIONS.each do |key, type|
+        env_value = ENV["FLUENT_OJ_OPTION_#{key.upcase}"]
+        next if env_value.nil?
+
+        cast_value = Fluent::Config.reformatted_value(OPTIONS[key], env_value, { strict: true })
+        next if cast_value.nil?
+
+        next if ALLOWED_VALUES[key] && !ALLOWED_VALUES[key].include?(cast_value)
+
+        options[key.to_sym] = cast_value
+      end
+
+      options
+    end
+  end
+end

data/lib/fluent/output.rb

@@ -0,0 +1,29 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/output'
+require 'fluent/output_chain'
+
+module Fluent
+  Output               = Fluent::Compat::Output
+  BufferedOutput       = Fluent::Compat::BufferedOutput
+  ObjectBufferedOutput = Fluent::Compat::ObjectBufferedOutput
+  TimeSlicedOutput     = Fluent::Compat::TimeSlicedOutput
+  MultiOutput          = Fluent::Compat::MultiOutput
+
+  # Some input plugins refer BufferQueueLimitError for throttling
+  BufferQueueLimitError = Fluent::Compat::BufferQueueLimitError
+end

data/lib/fluent/output_chain.rb

@@ -0,0 +1,23 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/output_chain'
+
+module Fluent
+  OutputChain = Fluent::Compat::OutputChain
+  CopyOutputChain = Fluent::Compat::CopyOutputChain
+  NullOutputChain = Fluent::Compat::NullOutputChain
+end

data/lib/fluent/parser.rb

@@ -0,0 +1,23 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/parser'
+
+module Fluent
+  ParserError = Fluent::Compat::Parser::ParserError
+  Parser = Fluent::Compat::Parser
+  TextParser = Fluent::Compat::TextParser
+end

data/lib/fluent/plugin/bare_output.rb

@@ -0,0 +1,104 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+
+require 'fluent/log'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Plugin
+    class BareOutput < Base
+      include PluginHelper::Mixin # for metrics
+
+      # DO NOT USE THIS plugin for normal output plugin. Use Output instead.
+      # This output plugin base class is only for meta-output plugins
+      # which cannot be implemented on MultiOutput.
+      # E.g,: forest, config-expander
+
+      helpers_internal :metrics
+
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin
+
+      def process(tag, es)
+        raise NotImplementedError, "BUG: output plugins MUST implement this method"
+      end
+
+      def num_errors
+        @num_errors_metrics.get
+      end
+
+      def emit_count
+        @emit_count_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
+      def initialize
+        super
+        @counter_mutex = Mutex.new
+        # TODO: well organized counters
+        @num_errors_metrics = nil
+        @emit_count_metrics = nil
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
+      end
+
+      def configure(conf)
+        super
+
+        @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "num_errors", help_text: "Number of count num errors")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of count emits")
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of emit records")
+        @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_size", help_text: "Total size of emit events")
+        @enable_size_metrics = !!system_config.enable_size_metrics
+      end
+
+      def statistics
+        stats = {
+          'num_errors' => @num_errors_metrics.get,
+          'emit_records' => @emit_records_metrics.get,
+          'emit_count' => @emit_count_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
+        }
+
+        { 'bare_output' => stats }
+      end
+
+      def emit_sync(tag, es)
+        @emit_count_metrics.inc
+        begin
+          process(tag, es)
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
+        rescue
+          @num_errors_metrics.inc
+          raise
+        end
+      end
+      alias :emit_events :emit_sync
+    end
+  end
+end