fluentd-hubspot 0.14.14.1

data/lib/fluent/match.rb

@@ -0,0 +1,178 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  class MatchPattern
+    def self.create(str)
+      if str == '**'
+        AllMatchPattern.new
+      else
+        GlobMatchPattern.new(str)
+      end
+    end
+  end
+
+  class AllMatchPattern < MatchPattern
+    def match(str)
+      true
+    end
+  end
+
+  class GlobMatchPattern < MatchPattern
+    def initialize(pat)
+      stack = []
+      regex = ['']
+      escape = false
+      dot = false
+
+      i = 0
+      while i < pat.length
+        c = pat[i,1]
+
+        if escape
+          regex.last << Regexp.escape(c)
+          escape = false
+          i += 1
+          next
+
+        elsif pat[i,2] == "**"
+          # recursive any
+          if dot
+            regex.last << "(?![^\\.])"
+            dot = false
+          end
+          if pat[i+2,1] == "."
+            regex.last << "(?:.*\\.|\\A)"
+            i += 3
+          else
+            regex.last << ".*"
+            i += 2
+          end
+          next
+
+        elsif dot
+          regex.last << "\\."
+          dot = false
+        end
+
+        if c == "\\"
+          escape = true
+
+        elsif c == "."
+          dot = true
+
+        elsif c == "*"
+          # any
+          regex.last << "[^\\.]*"
+
+          # TODO
+          #elsif c == "["
+          #  # character class
+          #  chars = ''
+          #  while i < pat.length
+          #    c = pat[i,1]
+          #    if c == "]"
+          #      break
+          #    else
+          #      chars << c
+          #    end
+          #    i += 1
+          #  end
+          #  regex.last << '['+Regexp.escape(chars).gsub("\\-",'-')+']'
+
+        elsif c == "{"
+          # or
+          stack.push []
+          regex.push ''
+
+        elsif c == "}" && !stack.empty?
+          stack.last << regex.pop
+          regex.last << Regexp.union(*stack.pop.map {|r| Regexp.new(r) }).to_s
+
+        elsif c == "," && !stack.empty?
+          stack.last << regex.pop
+          regex.push ''
+
+        elsif c =~ /[a-zA-Z0-9_]/
+          regex.last << c
+
+        else
+          regex.last << "\\#{c}"
+        end
+
+        i += 1
+      end
+
+      until stack.empty?
+        stack.last << regex.pop
+        regex.last << Regexp.union(*stack.pop).to_s
+      end
+
+      @regex = Regexp.new("\\A"+regex.last+"\\Z")
+    end
+
+    def match(str)
+      @regex.match(str) != nil
+    end
+  end
+
+  class OrMatchPattern < MatchPattern
+    def initialize(patterns)
+      @patterns = patterns
+    end
+
+    def match(str)
+      @patterns.any? {|pattern| pattern.match(str) }
+    end
+  end
+
+  class NoMatchMatch
+    def initialize(log)
+      @log = log
+      @count = 0
+      @warn_not_matched = true
+    end
+
+    def suppress_missing_match!
+      # for <label @FLUENT_LOG>
+      @warn_not_matched = false
+    end
+
+    def emit_events(tag, es)
+      return unless @warn_not_matched
+      # TODO use time instead of num of records
+      c = (@count += 1)
+      if c < 512
+        if Math.log(c) / Math.log(2) % 1.0 == 0
+          @log.warn "no patterns matched", tag: tag
+          return
+        end
+      else
+        if c % 512 == 0
+          @log.warn "no patterns matched", tag: tag
+          return
+        end
+      end
+      @log.on_trace { @log.trace "no patterns matched", tag: tag }
+    end
+
+    def start
+    end
+
+    def shutdown
+    end
+  end
+end

data/lib/fluent/mixin.rb

@@ -0,0 +1,31 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/record_filter_mixin'
+require 'fluent/compat/handle_tag_name_mixin'
+require 'fluent/compat/set_time_key_mixin'
+require 'fluent/compat/set_tag_key_mixin'
+require 'fluent/compat/type_converter'
+
+require 'fluent/time' # Fluent::TimeFormatter
+
+module Fluent
+  RecordFilterMixin = Fluent::Compat::RecordFilterMixin
+  HandleTagNameMixin = Fluent::Compat::HandleTagNameMixin
+  SetTimeKeyMixin = Fluent::Compat::SetTimeKeyMixin
+  SetTagKeyMixin = Fluent::Compat::SetTagKeyMixin
+  TypeConverter = Fluent::Compat::TypeConverter
+end

data/lib/fluent/msgpack_factory.rb

@@ -0,0 +1,62 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'msgpack'
+require 'fluent/time'
+
+module Fluent
+  module MessagePackFactory
+    @@engine_factory = nil
+
+    module Mixin
+      def msgpack_factory
+        MessagePackFactory.engine_factory
+      end
+
+      def msgpack_packer(*args)
+        msgpack_factory.packer(*args)
+      end
+
+      def msgpack_unpacker(*args)
+        msgpack_factory.unpacker(*args)
+      end
+    end
+
+    def self.engine_factory
+      @@engine_factory || factory
+    end
+
+    def self.factory
+      factory = MessagePack::Factory.new
+      factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)
+      factory
+    end
+
+    def self.packer(*args)
+      factory.packer(*args)
+    end
+
+    def self.unpacker(*args)
+      factory.unpacker(*args)
+    end
+
+    def self.init
+      factory = MessagePack::Factory.new
+      factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)
+      @@engine_factory = factory
+    end
+  end
+end

data/lib/fluent/output.rb

@@ -0,0 +1,29 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/output'
+require 'fluent/output_chain'
+
+module Fluent
+  Output               = Fluent::Compat::Output
+  BufferedOutput       = Fluent::Compat::BufferedOutput
+  ObjectBufferedOutput = Fluent::Compat::ObjectBufferedOutput
+  TimeSlicedOutput     = Fluent::Compat::TimeSlicedOutput
+  MultiOutput          = Fluent::Compat::MultiOutput
+
+  # Some input plugins refer BufferQueueLimitError for throttling
+  BufferQueueLimitError = Fluent::Compat::BufferQueueLimitError
+end

data/lib/fluent/output_chain.rb

@@ -0,0 +1,23 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/output_chain'
+
+module Fluent
+  OutputChain = Fluent::Compat::OutputChain
+  CopyOutputChain = Fluent::Compat::CopyOutputChain
+  NullOutputChain = Fluent::Compat::NullOutputChain
+end

data/lib/fluent/parser.rb

@@ -0,0 +1,23 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/parser'
+
+module Fluent
+  ParserError = Fluent::Compat::Parser::ParserError
+  Parser = Fluent::Compat::Parser
+  TextParser = Fluent::Compat::TextParser
+end

data/lib/fluent/plugin.rb

@@ -0,0 +1,183 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/registry'
+require 'fluent/config/error'
+
+module Fluent
+  module Plugin
+    SEARCH_PATHS = []
+
+    # plugins for fluentd:         fluent/plugin/type_NAME.rb
+    # plugins for fluentd plugins: fluent/plugin/type/NAME.rb
+    #   ex: storage, buffer chunk, ...
+
+    # first class plugins (instantiated by Engine)
+    INPUT_REGISTRY     = Registry.new(:input,     'fluent/plugin/in_',         dir_search_prefix: 'in_')
+    OUTPUT_REGISTRY    = Registry.new(:output,    'fluent/plugin/out_',        dir_search_prefix: 'out_')
+    FILTER_REGISTRY    = Registry.new(:filter,    'fluent/plugin/filter_',     dir_search_prefix: 'filter_')
+
+    # feature plugin: second class plugins (instantiated by Plugins or Helpers)
+    BUFFER_REGISTRY    = Registry.new(:buffer,    'fluent/plugin/buf_',        dir_search_prefix: 'buf_')
+    PARSER_REGISTRY    = Registry.new(:parser,    'fluent/plugin/parser_',     dir_search_prefix: 'parser_')
+    FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_',  dir_search_prefix: 'formatter_')
+    STORAGE_REGISTRY   = Registry.new(:storage,   'fluent/plugin/storage_',    dir_search_prefix: 'storage_')
+
+    REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY]
+
+    def self.register_input(type, klass)
+      register_impl('input', INPUT_REGISTRY, type, klass)
+    end
+
+    def self.register_output(type, klass)
+      register_impl('output', OUTPUT_REGISTRY, type, klass)
+    end
+
+    def self.register_filter(type, klass)
+      register_impl('filter', FILTER_REGISTRY, type, klass)
+    end
+
+    def self.register_buffer(type, klass)
+      register_impl('buffer', BUFFER_REGISTRY, type, klass)
+    end
+
+    def self.register_parser(type, klass_or_proc)
+      if klass_or_proc.is_a?(Regexp)
+        # This usage is not recommended for new API
+        require 'fluent/parser'
+        register_impl('parser', PARSER_REGISTRY, type, Proc.new { Fluent::TextParser::RegexpParser.new(klass_or_proc) })
+      else
+        register_impl('parser', PARSER_REGISTRY, type, klass_or_proc)
+      end
+    end
+
+    def self.register_formatter(type, klass_or_proc)
+      if klass_or_proc.respond_to?(:call) && klass_or_proc.arity == 3 # Proc.new { |tag, time, record| }
+        # This usage is not recommended for new API
+        require 'fluent/formatter'
+        register_impl('formatter', FORMATTER_REGISTRY, type, Proc.new { Fluent::TextFormatter::ProcWrappedFormatter.new(klass_or_proc) })
+      else
+        register_impl('formatter', FORMATTER_REGISTRY, type, klass_or_proc)
+      end
+    end
+
+    def self.register_storage(type, klass)
+      register_impl('storage', STORAGE_REGISTRY, type, klass)
+    end
+
+    def self.lookup_type_from_class(klass_or_its_name)
+      klass = if klass_or_its_name.is_a? Class
+                klass_or_its_name
+              elsif klass_or_its_name.is_a? String
+                eval(klass_or_its_name) # const_get can't handle qualified klass name (ex: A::B)
+              else
+                raise ArgumentError, "invalid argument type #{klass_or_its_name.class}: #{klass_or_its_name}"
+              end
+      REGISTRIES.reduce(nil){|a, r| a || r.reverse_lookup(klass) }
+    end
+
+    def self.add_plugin_dir(dir)
+      REGISTRIES.each do |r|
+        r.paths.push(dir)
+      end
+      nil
+    end
+
+    def self.new_input(type)
+      new_impl('input', INPUT_REGISTRY, type)
+    end
+
+    def self.new_output(type)
+      new_impl('output', OUTPUT_REGISTRY, type)
+    end
+
+    def self.new_filter(type)
+      new_impl('filter', FILTER_REGISTRY, type)
+    end
+
+    def self.new_buffer(type, parent: nil)
+      new_impl('buffer', BUFFER_REGISTRY, type, parent)
+    end
+
+    def self.new_parser(type, parent: nil)
+      if type[0] == '/' && type[-1] == '/'
+        # This usage is not recommended for new API... create RegexpParser directly
+        require 'fluent/parser'
+        impl = Fluent::TextParser.lookup(type)
+        impl.extend FeatureAvailabilityChecker
+        impl
+      else
+        new_impl('parser', PARSER_REGISTRY, type, parent)
+      end
+    end
+
+    def self.new_formatter(type, parent: nil)
+      new_impl('formatter', FORMATTER_REGISTRY, type, parent)
+    end
+
+    def self.new_storage(type, parent: nil)
+      new_impl('storage', STORAGE_REGISTRY, type, parent)
+    end
+
+    def self.register_impl(kind, registry, type, value)
+      if !value.is_a?(Class) && !value.respond_to?(:call)
+        raise Fluent::ConfigError, "Invalid implementation as #{kind} plugin: '#{type}'. It must be a Class, or callable."
+      end
+      registry.register(type, value)
+      $log.trace "registered #{kind} plugin '#{type}'" if defined?($log)
+      nil
+    end
+
+    def self.new_impl(kind, registry, type, parent=nil)
+      # "'type' not found" is handled by registry
+      obj = registry.lookup(type)
+      impl = case
+             when obj.is_a?(Class)
+               obj.new
+             when obj.respond_to?(:call) && obj.arity == 0
+               obj.call
+             else
+               raise Fluent::ConfigError, "#{kind} plugin '#{type}' is not a Class nor callable (without arguments)."
+             end
+      if parent && impl.respond_to?("owner=")
+        impl.owner = parent
+      end
+      impl.extend FeatureAvailabilityChecker
+      impl
+    end
+
+    module FeatureAvailabilityChecker
+      def configure(conf)
+        super
+
+        # extend plugin instance by this module
+        # to run this check after all #configure methods of plugins and plugin helpers
+        sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
+                    owner.system_config
+                  elsif self.respond_to?(:system_config)
+                    self.system_config
+                  else
+                    nil
+                  end
+
+        if sysconf && sysconf.workers > 1 && !self.multi_workers_ready?
+          type = Fluent::Plugin.lookup_type_from_class(self.class)
+          raise Fluent::ConfigError, "Plugin '#{type}' does not support multi workers configuration (#{self.class})"
+        end
+      end
+    end
+  end
+end