fluentd-hubspot 0.14.14.1

data/lib/fluent/env.rb

@@ -0,0 +1,28 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'serverengine/utils'
+
+module Fluent
+  DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
+  DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
+  DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
+  DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat, use_to_json: true}
+
+  def self.windows?
+    ServerEngine.windows?
+  end
+end

data/lib/fluent/error.rb

@@ -0,0 +1,30 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  class UnrecoverableError < StandardError
+    def initialize(error_message = nil)
+      @message = error_message || "an unrecoverable error occurs in Fluentd process"
+    end
+
+    def to_s
+      @message
+    end
+  end
+
+  class InvalidRootDirectory < UnrecoverableError
+  end
+end

data/lib/fluent/event.rb

@@ -0,0 +1,334 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/msgpack_factory'
+require 'fluent/plugin/compressable'
+
+module Fluent
+  class EventStream
+    include Enumerable
+    include MessagePackFactory::Mixin
+    include Fluent::Plugin::Compressable
+
+    # dup does deep copy for event stream
+    def dup
+      raise NotImplementedError, "DO NOT USE THIS CLASS directly."
+    end
+
+    def size
+      raise NotImplementedError, "DO NOT USE THIS CLASS directly."
+    end
+    alias :length :size
+
+    def empty?
+      size == 0
+    end
+
+    # for tests
+    def ==(other)
+      other.is_a?(EventStream) && self.to_msgpack_stream == other.to_msgpack_stream
+    end
+
+    def repeatable?
+      false
+    end
+
+    def slice(index, num)
+      raise NotImplementedError, "DO NOT USE THIS CLASS directly."
+    end
+
+    def each(&block)
+      raise NotImplementedError, "DO NOT USE THIS CLASS directly."
+    end
+
+    def to_msgpack_stream(time_int: false)
+      return to_msgpack_stream_forced_integer if time_int
+      out = msgpack_packer
+      each {|time,record|
+        out.write([time,record])
+      }
+      out.to_s
+    end
+
+    def to_compressed_msgpack_stream(time_int: false)
+      packed = to_msgpack_stream(time_int: time_int)
+      compress(packed)
+    end
+
+    def to_msgpack_stream_forced_integer
+      out = msgpack_packer
+      each {|time,record|
+        out.write([time.to_i,record])
+      }
+      out.to_s
+    end
+  end
+
+  class OneEventStream < EventStream
+    def initialize(time, record)
+      @time = time
+      @record = record
+    end
+
+    def dup
+      OneEventStream.new(@time, @record.dup)
+    end
+
+    def size
+      1
+    end
+
+    def repeatable?
+      true
+    end
+
+    def slice(index, num)
+      if index > 0 || num == 0
+        ArrayEventStream.new([])
+      else
+        self.dup
+      end
+    end
+
+    def each(&block)
+      block.call(@time, @record)
+      nil
+    end
+  end
+
+  # EventStream from entries: Array of [time, record]
+  #
+  # Use this class for many events data with a tag
+  # and its representation is [ [time, record], [time, record], .. ]
+  class ArrayEventStream < EventStream
+    def initialize(entries)
+      @entries = entries
+    end
+
+    def dup
+      entries = @entries.map{ |time, record| [time, record.dup] }
+      ArrayEventStream.new(entries)
+    end
+
+    def size
+      @entries.size
+    end
+
+    def repeatable?
+      true
+    end
+
+    def empty?
+      @entries.empty?
+    end
+
+    def slice(index, num)
+      ArrayEventStream.new(@entries.slice(index, num))
+    end
+
+    def each(&block)
+      @entries.each(&block)
+      nil
+    end
+  end
+
+  # EventStream from entries: numbers of pairs of time and record.
+  #
+  # This class can handle many events more efficiently than ArrayEventStream
+  # because this class generate less objects than ArrayEventStream.
+  #
+  # Use this class as below, in loop of data-enumeration:
+  #  1. initialize blank stream:
+  #     streams[tag] ||= MultiEventStream.new
+  #  2. add events
+  #     stream[tag].add(time, record)
+  class MultiEventStream < EventStream
+    def initialize(time_array = [], record_array = [])
+      @time_array = time_array
+      @record_array = record_array
+    end
+
+    def dup
+      MultiEventStream.new(@time_array.dup, @record_array.map(&:dup))
+    end
+
+    def size
+      @time_array.size
+    end
+
+    def add(time, record)
+      @time_array << time
+      @record_array << record
+    end
+
+    def repeatable?
+      true
+    end
+
+    def empty?
+      @time_array.empty?
+    end
+
+    def slice(index, num)
+      MultiEventStream.new(@time_array.slice(index, num), @record_array.slice(index, num))
+    end
+
+    def each(&block)
+      time_array = @time_array
+      record_array = @record_array
+      for i in 0..time_array.length-1
+        block.call(time_array[i], record_array[i])
+      end
+      nil
+    end
+  end
+
+  class MessagePackEventStream < EventStream
+    # https://github.com/msgpack/msgpack-ruby/issues/119
+
+    # Keep cached_unpacker argument for existing plugins
+    def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)
+      @data = data
+      @size = size
+      @unpacked_times = unpacked_times
+      @unpacked_records = unpacked_records
+    end
+
+    def empty?
+      @data.empty?
+    end
+
+    def dup
+      if @unpacked_times
+        self.class.new(@data.dup, nil, @size, unpacked_times: @unpacked_times, unpacked_records: @unpacked_records.map(&:dup))
+      else
+        self.class.new(@data.dup, nil, @size)
+      end
+    end
+
+    def size
+      # @size is unbelievable always when @size == 0
+      # If the number of events is really zero, unpacking events takes very short time.
+      ensure_unpacked! if @size == 0
+      @size
+    end
+
+    def repeatable?
+      true
+    end
+
+    def ensure_unpacked!
+      return if @unpacked_times && @unpacked_records
+      @unpacked_times = []
+      @unpacked_records = []
+      msgpack_unpacker.feed_each(@data) do |time, record|
+        @unpacked_times << time
+        @unpacked_records << record
+      end
+      # @size should be updated always right after unpack.
+      # The real size of unpacked objects are correct, rather than given size.
+      @size = @unpacked_times.size
+    end
+
+    # This method returns MultiEventStream, because there are no reason
+    # to surve binary serialized by msgpack.
+    def slice(index, num)
+      ensure_unpacked!
+      MultiEventStream.new(@unpacked_times.slice(index, num), @unpacked_records.slice(index, num))
+    end
+
+    def each(&block)
+      if @unpacked_times
+        @unpacked_times.each_with_index do |time, i|
+          block.call(time, @unpacked_records[i])
+        end
+      else
+        @unpacked_times = []
+        @unpacked_records = []
+        msgpack_unpacker.feed_each(@data) do |time, record|
+          @unpacked_times << time
+          @unpacked_records << record
+          block.call(time, record)
+        end
+        @size = @unpacked_times.size
+      end
+      nil
+    end
+
+    def to_msgpack_stream(time_int: false)
+      # time_int is always ignored because @data is always packed binary in this class
+      @data
+    end
+  end
+
+  class CompressedMessagePackEventStream < MessagePackEventStream
+    def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)
+      super
+      @decompressed_data = nil
+      @compressed_data = data
+    end
+
+    def empty?
+      ensure_decompressed!
+      super
+    end
+
+    def ensure_unpacked!
+      ensure_decompressed!
+      super
+    end
+
+    def each(&block)
+      ensure_decompressed!
+      super
+    end
+
+    def to_msgpack_stream(time_int: false)
+      ensure_decompressed!
+      super
+    end
+
+    def to_compressed_msgpack_stream(time_int: false)
+      # time_int is always ignored because @data is always packed binary in this class
+      @compressed_data
+    end
+
+    private
+
+    def ensure_decompressed!
+      return if @decompressed_data
+      @data = @decompressed_data = decompress(@data)
+    end
+  end
+
+  module ChunkMessagePackEventStreamer
+    include MessagePackFactory::Mixin
+    # chunk.extend(ChunkEventStreamer)
+    #  => chunk.each{|time, record| ... }
+    def each(&block)
+      open do |io|
+        msgpack_unpacker(io).each(&block)
+      end
+      nil
+    end
+    alias :msgpack_each :each
+
+    def to_msgpack_stream(time_int: false)
+      # time_int is always ignored because data is already packed and written in chunk
+      read
+    end
+  end
+end

data/lib/fluent/event_router.rb

@@ -0,0 +1,269 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/match'
+require 'fluent/event'
+require 'fluent/filter'
+
+module Fluent
+  #
+  # EventRouter is responsible to route events to a collector.
+  #
+  # It has a list of MatchPattern and Collector pairs:
+  #
+  #  +----------------+     +-----------------+
+  #  |  MatchPattern  |     |    Collector    |
+  #  +----------------+     +-----------------+
+  #  |   access.**  ---------> type forward   |
+  #  |     logs.**  ---------> type copy      |
+  #  |  archive.**  ---------> type s3        |
+  #  +----------------+     +-----------------+
+  #
+  # EventRouter does:
+  #
+  # 1) receive an event at `#emit` methods
+  # 2) match the event's tag with the MatchPatterns
+  # 3) forward the event to the corresponding Collector
+  #
+  # Collector is either of Output, Filter or other EventRouter.
+  #
+  class EventRouter
+    def initialize(default_collector, emit_error_handler)
+      @match_rules = []
+      @match_cache = MatchCache.new
+      @default_collector = default_collector
+      @emit_error_handler = emit_error_handler
+    end
+
+    attr_accessor :default_collector
+    attr_accessor :emit_error_handler
+
+    class Rule
+      def initialize(pattern, collector)
+        patterns = pattern.split(/\s+/).map { |str| MatchPattern.create(str) }
+        @pattern = if patterns.length == 1
+                     patterns[0]
+                   else
+                     OrMatchPattern.new(patterns)
+                   end
+        @pattern_str = pattern
+        @collector = collector
+      end
+
+      def match?(tag)
+        @pattern.match(tag)
+      end
+
+      attr_reader :collector
+      attr_reader :pattern_str
+    end
+
+    def suppress_missing_match!
+      if @default_collector.respond_to?(:suppress_missing_match!)
+        @default_collector.suppress_missing_match!
+      end
+    end
+
+    # called by Agent to add new match pattern and collector
+    def add_rule(pattern, collector)
+      @match_rules << Rule.new(pattern, collector)
+    end
+
+    def emit(tag, time, record)
+      unless record.nil?
+        emit_stream(tag, OneEventStream.new(time, record))
+      end
+    end
+
+    def emit_array(tag, array)
+      emit_stream(tag, ArrayEventStream.new(array))
+    end
+
+    def emit_stream(tag, es)
+      match(tag).emit_events(tag, es)
+    rescue => e
+      @emit_error_handler.handle_emits_error(tag, es, e)
+    end
+
+    def emit_error_event(tag, time, record, error)
+      @emit_error_handler.emit_error_event(tag, time, record, error)
+    end
+
+    def match?(tag)
+      !!find(tag)
+    end
+
+    def match(tag)
+      collector = @match_cache.get(tag) {
+        find(tag) || @default_collector
+      }
+      collector
+    end
+
+    class MatchCache
+      MATCH_CACHE_SIZE = 1024
+
+      def initialize
+        super
+        @map = {}
+        @keys = []
+      end
+
+      def get(key)
+        if collector = @map[key]
+          return collector
+        end
+        collector = @map[key] = yield
+        if @keys.size >= MATCH_CACHE_SIZE
+          # expire the oldest key
+          @map.delete @keys.shift
+        end
+        @keys << key
+        collector
+      end
+    end
+
+    private
+
+    class Pipeline
+      def initialize
+        @filters = []
+        @output = nil
+        @optimizer = FilterOptimizer.new
+      end
+
+      def add_filter(filter)
+        @filters << filter
+        @optimizer.filters = @filters
+      end
+
+      def set_output(output)
+        @output = output
+      end
+
+      def emit_events(tag, es)
+        processed = @optimizer.filter_stream(tag, es)
+        @output.emit_events(tag, processed)
+      end
+
+      class FilterOptimizer
+        def initialize(filters = [])
+          @filters = filters
+          @optimizable = nil
+        end
+
+        def filters=(filters)
+          @filters = filters
+          reset_optimization
+        end
+
+        def filter_stream(tag, es)
+          if optimizable?
+            optimized_filter_stream(tag, es)
+          else
+            @filters.reduce(es) { |acc, filter| filter.filter_stream(tag, acc) }
+          end
+        end
+
+        private
+
+        def optimized_filter_stream(tag, es)
+          new_es = MultiEventStream.new
+          es.each do |time, record|
+            filtered_record = record
+            filtered_time = time
+
+            catch :break_loop do
+              @filters.each do |filter|
+                if filter.has_filter_with_time
+                  begin
+                    filtered_time, filtered_record = filter.filter_with_time(tag, filtered_time, filtered_record)
+                    throw :break_loop unless filtered_record && filtered_time
+                  rescue => e
+                    filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
+                  end
+                else
+                  begin
+                    filtered_record = filter.filter(tag, filtered_time, filtered_record)
+                    throw :break_loop unless filtered_record
+                  rescue => e
+                    filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
+                  end
+                end
+              end
+
+              new_es.add(filtered_time, filtered_record)
+            end
+          end
+          new_es
+        end
+
+        def optimizable?
+          return @optimizable unless @optimizable.nil?
+          fs_filters = filters_having_filter_stream
+          @optimizable = if fs_filters.empty?
+                           true
+                         else
+                           # skip log message when filter is only 1, because its performace is same as non optimized chain.
+                           if @filters.size > 1 && fs_filters.size >= 1
+                             $log.info "disable filter chain optimization because #{fs_filters.map(&:class)} uses `#filter_stream` method."
+                           end
+                           false
+                         end
+        end
+
+        def filters_having_filter_stream
+          @filters_having_filter_stream ||= @filters.select do |filter|
+            filter.class.instance_methods(false).include?(:filter_stream)
+          end
+        end
+
+        def reset_optimization
+          @optimizable = nil
+          @filters_having_filter_stream = nil
+        end
+      end
+    end
+
+    def find(tag)
+      pipeline = nil
+      @match_rules.each_with_index { |rule, i|
+        if rule.match?(tag)
+          if rule.collector.is_a?(Plugin::Filter)
+            pipeline ||= Pipeline.new
+            pipeline.add_filter(rule.collector)
+          else
+            if pipeline
+              pipeline.set_output(rule.collector)
+            else
+              # Use Output directly when filter is not matched
+              pipeline = rule.collector
+            end
+            return pipeline
+          end
+        end
+      }
+
+      if pipeline
+        # filter is matched but no match
+        pipeline.set_output(@default_collector)
+        pipeline
+      else
+        nil
+      end
+    end
+  end
+end