fluentd-hubspot 0.14.14.1

data/lib/fluent/test/startup_shutdown.rb

@@ -0,0 +1,46 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'serverengine'
+require 'fileutils'
+
+module Fluent
+  module Test
+    module StartupShutdown
+      def startup
+        socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+        @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
+        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+      end
+
+      def shutdown
+        @server.close
+      end
+
+      def self.setup
+        @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+        @server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
+        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
+      end
+
+      def self.teardown
+        @server.close
+        # on Windows, socket_manager_path is a TCP port number
+        FileUtils.rm_f @socket_manager_path unless Fluent.windows?
+      end
+    end
+  end
+end

data/lib/fluent/time.rb

@@ -0,0 +1,412 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'time'
+require 'msgpack'
+require 'strptime'
+require 'fluent/timezone'
+require 'fluent/configurable'
+require 'fluent/config/error'
+
+module Fluent
+  class EventTime
+    TYPE = 0
+
+    def initialize(sec, nsec = 0)
+      @sec = sec
+      @nsec = nsec
+    end
+
+    def ==(other)
+      if other.is_a?(Fluent::EventTime)
+        @sec == other.sec
+      else
+        @sec == other
+      end
+    end
+
+    def sec
+      @sec
+    end
+
+    def nsec
+      @nsec
+    end
+
+    def to_int
+      @sec
+    end
+
+    def to_f
+      @sec + @nsec / 1_000_000_000.0
+    end
+
+    # for Time.at
+    def to_r
+      Rational(@sec * 1_000_000_000 + @nsec, 1_000_000_000)
+    end
+
+    # for > and others
+    def coerce(other)
+      [other, @sec]
+    end
+
+    def to_s
+      @sec.to_s
+    end
+
+    def to_json(*args)
+      @sec.to_s
+    end
+
+    def to_msgpack(io = nil)
+      @sec.to_msgpack(io)
+    end
+
+    def to_msgpack_ext
+      [@sec, @nsec].pack('NN')
+    end
+
+    def self.from_msgpack_ext(data)
+      new(*data.unpack('NN'))
+    end
+
+    def self.from_time(time)
+      Fluent::EventTime.new(time.to_i, time.nsec)
+    end
+
+    def self.eq?(a, b)
+      if a.is_a?(Fluent::EventTime) && b.is_a?(Fluent::EventTime)
+        a.sec == b.sec && a.nsec == b.nsec
+      else
+        a == b
+      end
+    end
+
+    def self.now
+      from_time(Time.now)
+    end
+
+    def self.parse(*args)
+      from_time(Time.parse(*args))
+    end
+
+    ## TODO: For performance, implement +, -, and so on
+    def method_missing(name, *args, &block)
+      @sec.send(name, *args, &block)
+    end
+  end
+
+  module TimeMixin
+    TIME_TYPES = ['string', 'unixtime', 'float']
+
+    TIME_PARAMETERS = [
+      [:time_format, :string, {default: nil}],
+      [:localtime, :bool, {default: true}],  # UTC if :localtime is false and :timezone is nil
+      [:utc,       :bool, {default: false}], # to turn :localtime false
+      [:timezone, :string, {default: nil}],
+    ]
+    TIME_FULL_PARAMETERS = [
+      # To avoid to define :time_type twice (in plugin_helper/inject)
+      [:time_type, :enum, {default: :string, list: TIME_TYPES.map(&:to_sym)}],
+    ] + TIME_PARAMETERS
+
+    module TimeParameters
+      include Fluent::Configurable
+      TIME_FULL_PARAMETERS.each do |name, type, opts|
+        config_param name, type, opts
+      end
+
+      def configure(conf)
+        if conf.has_key?('localtime') || conf.has_key?('utc')
+          if conf.has_key?('localtime') && conf.has_key?('utc')
+            raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
+          elsif conf.has_key?('localtime')
+            conf['localtime'] = Fluent::Config.bool_value(conf['localtime'])
+          elsif conf.has_key?('utc')
+            conf['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
+            # Specifying "localtime false" means using UTC in TimeFormatter
+            # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
+            # There are difference between "Z" and "+0000" in timezone formatting.
+            # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
+          end
+        end
+
+        super
+
+        Fluent::Timezone.validate!(@timezone) if @timezone
+      end
+    end
+
+    module Parser
+      def self.included(mod)
+        mod.include TimeParameters
+      end
+
+      def time_parser_create(type: @time_type, format: @time_format, timezone: @timezone, force_localtime: false)
+        return NumericTimeParser.new(type) if type != :string
+        return TimeParser.new(format, true, nil) if force_localtime
+
+        localtime = @localtime && (timezone.nil? && !@utc)
+        TimeParser.new(format, localtime, timezone)
+      end
+    end
+
+    module Formatter
+      def self.included(mod)
+        mod.include TimeParameters
+      end
+
+      def time_formatter_create(type: @time_type, format: @time_format, timezone: @timezone, force_localtime: false)
+        return NumericTimeFormatter.new(type) if type != :string
+        return TimeFormatter.new(format, true, nil) if force_localtime
+
+        localtime = @localtime && (timezone.nil? && !@utc)
+        TimeFormatter.new(format, localtime, timezone)
+      end
+    end
+  end
+
+  class TimeParser
+    class TimeParseError < StandardError; end
+
+    def initialize(format = nil, localtime = true, timezone = nil)
+      if format.nil? && (timezone || !localtime)
+        raise Fluent::ConfigError, "specifying timezone requires time format"
+      end
+
+      @cache1_key = nil
+      @cache1_time = nil
+      @cache2_key = nil
+      @cache2_time = nil
+
+      format_with_timezone = format && (format.include?("%z") || format.include?("%Z"))
+
+      # unixtime_in_expected_tz = unixtime_in_localtime + offset_diff
+      offset_diff = case
+                    when format_with_timezone then nil
+                    when timezone  then Time.now.localtime.utc_offset - Time.zone_offset(timezone)
+                    when localtime then 0
+                    else Time.now.localtime.utc_offset # utc
+                    end
+
+      strptime = format && (Strptime.new(format) rescue nil)
+
+      @parse = case
+               when format_with_timezone && strptime then ->(v){ Fluent::EventTime.from_time(strptime.exec(v)) }
+               when format_with_timezone             then ->(v){ Fluent::EventTime.from_time(Time.strptime(v, format)) }
+               when strptime then ->(v){ t = strptime.exec(v);         Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
+               when format   then ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
+               else ->(v){ Fluent::EventTime.parse(v) }
+               end
+    end
+
+    # TODO: new cache mechanism using format string
+    def parse(value)
+      unless value.is_a?(String)
+        raise TimeParseError, "value must be string: #{value}"
+      end
+
+      if @cache1_key == value
+        return @cache1_time
+      elsif @cache2_key == value
+        return @cache2_time
+      else
+        begin
+          time = @parse.call(value)
+        rescue => e
+          raise TimeParseError, "invalid time format: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
+        end
+        @cache1_key = @cache2_key
+        @cache1_time = @cache2_time
+        @cache2_key = value
+        @cache2_time = time
+        return time
+      end
+    end
+    alias :call :parse
+  end
+
+  class NumericTimeParser < TimeParser # to include TimeParseError
+    def initialize(type, localtime = nil, timezone = nil)
+      @cache1_key = @cache1_time = @cache2_key = @cache2_time = nil
+
+      if type == :unixtime
+        define_singleton_method(:parse, method(:parse_unixtime))
+        define_singleton_method(:call, method(:parse_unixtime))
+      else # :float
+        define_singleton_method(:parse, method(:parse_float))
+        define_singleton_method(:call, method(:parse_float))
+      end
+    end
+
+    def parse_unixtime(value)
+      unless value.is_a?(String) || value.is_a?(Numeric)
+        raise TimeParseError, "value must be a string or a number: #{value}(value.class)"
+      end
+
+      if @cache1_key == value
+        return @cache1_time
+      elsif @cache2_key == value
+        return @cache2_time
+      end
+
+      begin
+        time = Fluent::EventTime.new(value.to_i)
+      rescue => e
+        raise TimeParseError, "invalid time format: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
+      end
+      @cache1_key = @cache2_key
+      @cache1_time = @cache2_time
+      @cache2_key = value
+      @cache2_time = time
+      time
+    end
+
+    # rough benchmark result to compare handmade parser vs Fluent::EventTime.from_time(Time.at(value.to_r))
+    # full: with 9-digits of nsec after dot
+    # msec: with 3-digits of msec after dot
+    # 10_000_000 times loop on MacBookAir
+    ## parse_by_myself(full): 12.162475 sec
+    ## parse_by_myself(msec): 15.050435 sec
+    ## parse_by_to_r  (full): 28.722362 sec
+    ## parse_by_to_r  (msec): 28.232856 sec
+    def parse_float(value)
+      unless value.is_a?(String) || value.is_a?(Numeric)
+        raise TimeParseError, "value must be a string or a number: #{value}(value.class)"
+      end
+
+      if @cache1_key == value
+        return @cache1_time
+      elsif @cache2_key == value
+        return @cache2_time
+      end
+
+      begin
+        sec_s, nsec_s, _ = value.to_s.split('.', 3) # throw away second-dot and later
+        nsec_s = nsec_s && nsec_s[0..9] || '0'
+        nsec_s += '0' * (9 - nsec_s.size) if nsec_s.size < 9
+        time = Fluent::EventTime.new(sec_s.to_i, nsec_s.to_i)
+      rescue => e
+        raise TimeParseError, "invalid time format: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
+      end
+      @cache1_key = @cache2_key
+      @cache1_time = @cache2_time
+      @cache2_key = value
+      @cache2_time = time
+      time
+    end
+  end
+
+  class TimeFormatter
+    def initialize(format = nil, localtime = true, timezone = nil)
+      @tc1 = 0
+      @tc1_str = nil
+      @tc2 = 0
+      @tc2_str = nil
+
+      if format && format =~ /(^|[^%])(%%)*%L|(^|[^%])(%%)*%\d*N/
+        define_singleton_method(:format, method(:format_with_subsec))
+        define_singleton_method(:call, method(:format_with_subsec))
+      else
+        define_singleton_method(:format, method(:format_without_subsec))
+        define_singleton_method(:call, method(:format_with_subsec))
+      end
+
+      formatter = Fluent::Timezone.formatter(timezone, format)
+      @format_nocache = case
+                        when formatter           then formatter
+                        when format && localtime then ->(time){ Time.at(time).strftime(format) }
+                        when format              then ->(time){ Time.at(time).utc.strftime(format) }
+                        when localtime           then ->(time){ Time.at(time).iso8601 }
+                        else                          ->(time){ Time.at(time).utc.iso8601 }
+                        end
+    end
+
+    def format_without_subsec(time)
+      if @tc1 == time
+        return @tc1_str
+      elsif @tc2 == time
+        return @tc2_str
+      else
+        str = format_nocache(time)
+        if @tc1 < @tc2
+          @tc1 = time
+          @tc1_str = str
+        else
+          @tc2 = time
+          @tc2_str = str
+        end
+        return str
+      end
+    end
+
+    def format_with_subsec(time)
+      if Fluent::EventTime.eq?(@tc1, time)
+        return @tc1_str
+      elsif Fluent::EventTime.eq?(@tc2, time)
+        return @tc2_str
+      else
+        str = format_nocache(time)
+        if @tc1 < @tc2
+          @tc1 = time
+          @tc1_str = str
+        else
+          @tc2 = time
+          @tc2_str = str
+        end
+        return str
+      end
+    end
+
+    ## Dynamically defined in #initialize
+    # def format(time)
+    # end
+
+    def format_nocache(time)
+      @format_nocache.call(time)
+    end
+  end
+
+  class NumericTimeFormatter < TimeFormatter
+    def initialize(type, localtime = nil, timezone = nil)
+      @cache1_key = @cache1_time = @cache2_key = @cache2_time = nil
+
+      if type == :unixtime
+        define_singleton_method(:format, method(:format_unixtime))
+        define_singleton_method(:call, method(:format_unixtime))
+      else # :float
+        define_singleton_method(:format, method(:format_float))
+        define_singleton_method(:call, method(:format_float))
+      end
+    end
+
+    def format_unixtime(time)
+      time.to_i.to_s
+    end
+
+    def format_float(time)
+      if time.is_a?(Fluent::EventTime) || time.is_a?(Time)
+        # 10.015 secs for 10_000_000 times call on MacBookAir
+        nsec_s = time.nsec.to_s
+        nsec_s = '0' * (9 - nsec_s.size) if nsec_s.size < 9
+        "#{time.sec}.#{nsec_s}"
+      else # integer (or float?)
+        time.to_f.to_s
+      end
+    end
+  end
+end