fluentd 0.12.40 → 1.6.2

data/lib/fluent/time.rb

@@ -1 +1,450 @@
-# placeholder for v0.14 time module
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'time'
+require 'msgpack'
+require 'strptime'
+require 'fluent/timezone'
+require 'fluent/configurable'
+require 'fluent/config/error'
+
+module Fluent
+  class EventTime
+    TYPE = 0
+    FORMATTER = Strftime.new('%Y-%m-%d %H:%M:%S.%N %z')
+
+    def initialize(sec, nsec = 0)
+      @sec = sec
+      @nsec = nsec
+    end
+
+    def ==(other)
+      if other.is_a?(Fluent::EventTime)
+        @sec == other.sec
+      else
+        @sec == other
+      end
+    end
+
+    def sec
+      @sec
+    end
+
+    def nsec
+      @nsec
+    end
+
+    def to_int
+      @sec
+    end
+
+    def to_f
+      @sec + @nsec / 1_000_000_000.0
+    end
+
+    # for Time.at
+    def to_r
+      Rational(@sec * 1_000_000_000 + @nsec, 1_000_000_000)
+    end
+
+    # for > and others
+    def coerce(other)
+      [other, @sec]
+    end
+
+    def to_s
+      @sec.to_s
+    end
+
+    begin
+      # ruby 2.5 or later
+      Time.at(0, 0, :nanosecond)
+
+      def to_time
+        Time.at(@sec, @nsec, :nanosecond)
+      end
+    rescue
+      def to_time
+        Time.at(@sec, @nsec / 1000.0)
+      end
+    end
+
+    def to_json(*args)
+      @sec.to_s
+    end
+
+    def to_msgpack(io = nil)
+      @sec.to_msgpack(io)
+    end
+
+    def to_msgpack_ext
+      [@sec, @nsec].pack('NN')
+    end
+
+    def self.from_msgpack_ext(data)
+      new(*data.unpack('NN'))
+    end
+
+    def self.from_time(time)
+      Fluent::EventTime.new(time.to_i, time.nsec)
+    end
+
+    def self.eq?(a, b)
+      if a.is_a?(Fluent::EventTime) && b.is_a?(Fluent::EventTime)
+        a.sec == b.sec && a.nsec == b.nsec
+      else
+        a == b
+      end
+    end
+
+    def self.now
+      from_time(Time.now)
+    end
+
+    def self.parse(*args)
+      from_time(Time.parse(*args))
+    end
+
+    ## TODO: For performance, implement +, -, and so on
+    def method_missing(name, *args, &block)
+      @sec.send(name, *args, &block)
+    end
+
+    def inspect
+      FORMATTER.exec(Time.at(self))
+    end
+  end
+
+  module TimeMixin
+    TIME_TYPES = ['string', 'unixtime', 'float']
+
+    TIME_PARAMETERS = [
+      [:time_format, :string, {default: nil}],
+      [:localtime, :bool, {default: true}],  # UTC if :localtime is false and :timezone is nil
+      [:utc,       :bool, {default: false}], # to turn :localtime false
+      [:timezone, :string, {default: nil}],
+    ]
+    TIME_FULL_PARAMETERS = [
+      # To avoid to define :time_type twice (in plugin_helper/inject)
+      [:time_type, :enum, {default: :string, list: TIME_TYPES.map(&:to_sym)}],
+    ] + TIME_PARAMETERS
+
+    module TimeParameters
+      include Fluent::Configurable
+      TIME_FULL_PARAMETERS.each do |name, type, opts|
+        config_param name, type, opts
+      end
+
+      def configure(conf)
+        if conf.has_key?('localtime') || conf.has_key?('utc')
+          if conf.has_key?('localtime') && conf.has_key?('utc')
+            raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
+          elsif conf.has_key?('localtime')
+            conf['localtime'] = Fluent::Config.bool_value(conf['localtime'])
+          elsif conf.has_key?('utc')
+            conf['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
+            # Specifying "localtime false" means using UTC in TimeFormatter
+            # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
+            # There are difference between "Z" and "+0000" in timezone formatting.
+            # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
+          end
+        end
+
+        super
+
+        Fluent::Timezone.validate!(@timezone) if @timezone
+      end
+    end
+
+    module Parser
+      def self.included(mod)
+        mod.include TimeParameters
+      end
+
+      def time_parser_create(type: @time_type, format: @time_format, timezone: @timezone, force_localtime: false)
+        return NumericTimeParser.new(type) if type != :string
+        return TimeParser.new(format, true, nil) if force_localtime
+
+        localtime = @localtime && (timezone.nil? && !@utc)
+        TimeParser.new(format, localtime, timezone)
+      end
+    end
+
+    module Formatter
+      def self.included(mod)
+        mod.include TimeParameters
+      end
+
+      def time_formatter_create(type: @time_type, format: @time_format, timezone: @timezone, force_localtime: false)
+        return NumericTimeFormatter.new(type) if type != :string
+        return TimeFormatter.new(format, true, nil) if force_localtime
+
+        localtime = @localtime && (timezone.nil? && !@utc)
+        TimeFormatter.new(format, localtime, timezone)
+      end
+    end
+  end
+
+  class TimeParser
+    class TimeParseError < StandardError; end
+
+    def initialize(format = nil, localtime = true, timezone = nil)
+      if format.nil? && (timezone || !localtime)
+        raise Fluent::ConfigError, "specifying timezone requires time format"
+      end
+
+      @cache1_key = nil
+      @cache1_time = nil
+      @cache2_key = nil
+      @cache2_time = nil
+
+      format_with_timezone = format && (format.include?("%z") || format.include?("%Z"))
+
+      # unixtime_in_expected_tz = unixtime_in_localtime + offset_diff
+      offset_diff = case
+                    when format_with_timezone then nil
+                    when timezone  then
+                      offset = Fluent::Timezone.utc_offset(timezone)
+                      if offset.respond_to?(:call)
+                        ->(t) { Time.now.localtime.utc_offset - offset.call(t) }
+                      else
+                        Time.now.localtime.utc_offset - offset
+                      end
+                    when localtime then 0
+                    else Time.now.localtime.utc_offset # utc
+                    end
+
+      strptime = format && (Strptime.new(format) rescue nil)
+
+      @parse = case
+               when format_with_timezone && strptime then ->(v){ Fluent::EventTime.from_time(strptime.exec(v)) }
+               when format_with_timezone             then ->(v){ Fluent::EventTime.from_time(Time.strptime(v, format)) }
+               when format == '%iso8601'             then ->(v){ Fluent::EventTime.from_time(Time.iso8601(v)) }
+               when strptime then
+                 if offset_diff.respond_to?(:call)
+                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + offset_diff.call(t), t.nsec) }
+                 else
+                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
+                 end
+               when format   then
+                 if offset_diff.respond_to?(:call)
+                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + offset_diff.call(t), t.nsec) }
+                 else
+                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
+                 end
+               else ->(v){ Fluent::EventTime.parse(v) }
+               end
+    end
+
+    # TODO: new cache mechanism using format string
+    def parse(value)
+      unless value.is_a?(String)
+        raise TimeParseError, "value must be string: #{value}"
+      end
+
+      if @cache1_key == value
+        return @cache1_time
+      elsif @cache2_key == value
+        return @cache2_time
+      else
+        begin
+          time = @parse.call(value)
+        rescue => e
+          raise TimeParseError, "invalid time format: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
+        end
+        @cache1_key = @cache2_key
+        @cache1_time = @cache2_time
+        @cache2_key = value
+        @cache2_time = time
+        return time
+      end
+    end
+    alias :call :parse
+  end
+
+  class NumericTimeParser < TimeParser # to include TimeParseError
+    def initialize(type, localtime = nil, timezone = nil)
+      @cache1_key = @cache1_time = @cache2_key = @cache2_time = nil
+
+      if type == :unixtime
+        define_singleton_method(:parse, method(:parse_unixtime))
+        define_singleton_method(:call, method(:parse_unixtime))
+      else # :float
+        define_singleton_method(:parse, method(:parse_float))
+        define_singleton_method(:call, method(:parse_float))
+      end
+    end
+
+    def parse_unixtime(value)
+      unless value.is_a?(String) || value.is_a?(Numeric)
+        raise TimeParseError, "value must be a string or a number: #{value}(value.class)"
+      end
+
+      if @cache1_key == value
+        return @cache1_time
+      elsif @cache2_key == value
+        return @cache2_time
+      end
+
+      begin
+        time = Fluent::EventTime.new(value.to_i)
+      rescue => e
+        raise TimeParseError, "invalid time format: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
+      end
+      @cache1_key = @cache2_key
+      @cache1_time = @cache2_time
+      @cache2_key = value
+      @cache2_time = time
+      time
+    end
+
+    # rough benchmark result to compare handmade parser vs Fluent::EventTime.from_time(Time.at(value.to_r))
+    # full: with 9-digits of nsec after dot
+    # msec: with 3-digits of msec after dot
+    # 10_000_000 times loop on MacBookAir
+    ## parse_by_myself(full): 12.162475 sec
+    ## parse_by_myself(msec): 15.050435 sec
+    ## parse_by_to_r  (full): 28.722362 sec
+    ## parse_by_to_r  (msec): 28.232856 sec
+    def parse_float(value)
+      unless value.is_a?(String) || value.is_a?(Numeric)
+        raise TimeParseError, "value must be a string or a number: #{value}(value.class)"
+      end
+
+      if @cache1_key == value
+        return @cache1_time
+      elsif @cache2_key == value
+        return @cache2_time
+      end
+
+      begin
+        sec_s, nsec_s, _ = value.to_s.split('.', 3) # throw away second-dot and later
+        nsec_s = nsec_s && nsec_s[0..9] || '0'
+        nsec_s += '0' * (9 - nsec_s.size) if nsec_s.size < 9
+        time = Fluent::EventTime.new(sec_s.to_i, nsec_s.to_i)
+      rescue => e
+        raise TimeParseError, "invalid time format: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
+      end
+      @cache1_key = @cache2_key
+      @cache1_time = @cache2_time
+      @cache2_key = value
+      @cache2_time = time
+      time
+    end
+  end
+
+  class TimeFormatter
+    def initialize(format = nil, localtime = true, timezone = nil)
+      @tc1 = 0
+      @tc1_str = nil
+      @tc2 = 0
+      @tc2_str = nil
+
+      strftime = format && (Strftime.new(format) rescue nil)
+      if format && format =~ /(^|[^%])(%%)*%L|(^|[^%])(%%)*%\d*N/
+        define_singleton_method(:format, method(:format_with_subsec))
+        define_singleton_method(:call, method(:format_with_subsec))
+      else
+        define_singleton_method(:format, method(:format_without_subsec))
+        define_singleton_method(:call, method(:format_without_subsec))
+      end
+
+      formatter = Fluent::Timezone.formatter(timezone, strftime ? strftime : format)
+      @format_nocache = case
+                        when formatter             then formatter
+                        when strftime && localtime then ->(time){ strftime.exec(Time.at(time)) }
+                        when format && localtime   then ->(time){ Time.at(time).strftime(format) }
+                        when strftime              then ->(time){ strftime.exec(Time.at(time).utc) }
+                        when format                then ->(time){ Time.at(time).utc.strftime(format) }
+                        when localtime             then ->(time){ Time.at(time).iso8601 }
+                        else                            ->(time){ Time.at(time).utc.iso8601 }
+                        end
+    end
+
+    def format_without_subsec(time)
+      if @tc1 == time
+        return @tc1_str
+      elsif @tc2 == time
+        return @tc2_str
+      else
+        str = format_nocache(time)
+        if @tc1 < @tc2
+          @tc1 = time
+          @tc1_str = str
+        else
+          @tc2 = time
+          @tc2_str = str
+        end
+        return str
+      end
+    end
+
+    def format_with_subsec(time)
+      if Fluent::EventTime.eq?(@tc1, time)
+        return @tc1_str
+      elsif Fluent::EventTime.eq?(@tc2, time)
+        return @tc2_str
+      else
+        str = format_nocache(time)
+        if @tc1 < @tc2
+          @tc1 = time
+          @tc1_str = str
+        else
+          @tc2 = time
+          @tc2_str = str
+        end
+        return str
+      end
+    end
+
+    ## Dynamically defined in #initialize
+    # def format(time)
+    # end
+
+    def format_nocache(time)
+      @format_nocache.call(time)
+    end
+  end
+
+  class NumericTimeFormatter < TimeFormatter
+    def initialize(type, localtime = nil, timezone = nil)
+      @cache1_key = @cache1_time = @cache2_key = @cache2_time = nil
+
+      if type == :unixtime
+        define_singleton_method(:format, method(:format_unixtime))
+        define_singleton_method(:call, method(:format_unixtime))
+      else # :float
+        define_singleton_method(:format, method(:format_float))
+        define_singleton_method(:call, method(:format_float))
+      end
+    end
+
+    def format_unixtime(time)
+      time.to_i.to_s
+    end
+
+    def format_float(time)
+      if time.is_a?(Fluent::EventTime) || time.is_a?(Time)
+        # 10.015 secs for 10_000_000 times call on MacBookAir
+        nsec_s = time.nsec.to_s
+        nsec_s = '0' * (9 - nsec_s.size) if nsec_s.size < 9
+        "#{time.sec}.#{nsec_s}"
+      else # integer (or float?)
+        time.to_f.to_s
+      end
+    end
+  end
+end

data/lib/fluent/timezone.rb

@@ -88,7 +88,7 @@ module Fluent
     #
     # An Proc object is returned. If the given timezone is invalid,
     # nil is returned.
-    def self.formatter(timezone, format = nil)
+    def self.formatter(timezone = nil, format = nil)
       if timezone.nil?
         return nil
       end
@@ -97,10 +97,15 @@ module Fluent
       if NUMERIC_PATTERN === timezone
         offset = Time.zone_offset(timezone)
 
-        if format
+        case
+        when format.is_a?(String)
           return Proc.new {|time|
             Time.at(time).localtime(offset).strftime(format)
           }
+        when format.is_a?(Strftime)
+          return Proc.new {|time|
+            format.exec(Time.at(time).localtime(offset))
+          }
         else
           return Proc.new {|time|
             Time.at(time).localtime(offset).iso8601
@@ -116,10 +121,15 @@ module Fluent
           return nil
         end
 
-        if format
+        case
+        when format.is_a?(String)
           return Proc.new {|time|
             Time.at(time).localtime(tz.period_for_utc(time).utc_total_offset).strftime(format)
           }
+        when format.is_a?(Strftime)
+          return Proc.new {|time|
+            format.exec(Time.at(time).localtime(tz.period_for_utc(time).utc_total_offset))
+          }
         else
           return Proc.new {|time|
             Time.at(time).localtime(tz.period_for_utc(time).utc_total_offset).iso8601
@@ -129,5 +139,19 @@ module Fluent
 
       return nil
     end
+
+    def self.utc_offset(timezone)
+      return 0 if timezone.nil?
+
+      case timezone
+      when NUMERIC_PATTERN
+        Time.zone_offset(timezone)
+      when NAME_PATTERN
+        tz = TZInfo::Timezone.get(timezone)
+        ->(time) {
+          tz.period_for_utc(time).utc_total_offset
+        }
+      end
+    end
   end
 end

data/lib/fluent/{status.rb → unique_id.rb}

@@ -15,34 +15,25 @@
 #
 
 module Fluent
-  class StatusClass
-    def initialize
-      @entries = {}
-      @mutex = Mutex.new
+  module UniqueId
+    def self.generate
+      now = Time.now.utc
+      u1 = ((now.to_i * 1000 * 1000 + now.usec) << 12 | rand(0xfff))
+      [u1 >> 32, u1 & 0xffffffff, rand(0xffffffff), rand(0xffffffff)].pack('NNNN')
     end
 
-    def register(instance, name, &block)
-      @mutex.synchronize {
-        (@entries[instance.object_id] ||= {})[name] = block
-      }
-      nil
+    def self.hex(unique_id)
+      unique_id.unpack('H*').first
     end
 
-    def each(&block)
-      @mutex.synchronize {
-        @entries.each {|obj_id,hash|
-          record = {}
-          hash.each_pair {|name,block|
-            record[name] = block.call
-          }
-          block.call(record)
-        }
-      }
+    module Mixin
+      def generate_unique_id
+        Fluent::UniqueId.generate
+      end
+
+      def dump_unique_id_hex(unique_id)
+        Fluent::UniqueId.hex(unique_id)
+      end
     end
   end
-
-  # Don't use this class from plugins.
-  # The interface may be changed
-  Status = StatusClass.new
 end
-

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.40'
+  VERSION = '1.6.2'
 
 end

data/lib/fluent/winsvc.rb

@@ -0,0 +1,85 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+begin
+
+  require 'optparse'
+  require 'windows/debug'
+  require 'Windows/Library'
+  require 'win32/daemon'
+  require 'win32/event'
+
+  include Win32
+  include Windows::Library
+  include Windows::Debug
+
+  op = OptionParser.new
+  opts = {service_name: nil}
+  op.on('--service-name NAME', "The name of the Windows Service") {|name|
+    opts[:service_name] = name
+  }
+  op.parse(ARGV)
+  if opts[:service_name] == nil
+    raise "Error: No Windows Service name set. Use '--service-name'"
+  end 
+
+  def read_fluentdopt(service_name)
+    require 'win32/Registry'
+    Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\#{service_name}") do |reg|
+      reg.read("fluentdopt")[1] rescue ""
+    end
+  end
+
+  def service_main_start(service_name)
+    ruby_path = 0.chr * 260
+    GetModuleFileName.call(0, ruby_path,260)
+    ruby_path = ruby_path.rstrip.gsub(/\\/, '/')
+    rubybin_dir = ruby_path[0, ruby_path.rindex("/")]
+    opt = read_fluentdopt(service_name)
+    Process.spawn("\"#{rubybin_dir}/ruby.exe\" \"#{rubybin_dir}/fluentd\" #{opt} -x #{service_name}")
+  end
+
+  class FluentdService < Daemon
+    @pid = 0
+    @service_name = ''
+
+    def initialize(service_name)
+      @service_name = service_name
+    end
+    
+    def service_main
+    
+      @pid = service_main_start(@service_name)
+      while running?
+        sleep 10
+      end
+    end
+
+    def service_stop
+      ev = Win32::Event.open(@service_name)
+      ev.set
+      ev.close
+      if @pid > 0
+        Process.waitpid(@pid)
+      end
+    end
+  end
+
+  FluentdService.new(opts[:service_name]).mainloop
+
+rescue Exception => err
+  raise
+end

data/templates/new_gem/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/templates/new_gem/README.md.erb

@@ -0,0 +1,43 @@
+# <%= gem_name %>
+
+[Fluentd](https://fluentd.org/) <%= type %> plugin to do something.
+
+TODO: write description for you plugin.
+
+## Installation
+
+### RubyGems
+
+```
+$ gem install <%= gem_name %>
+```
+
+### Bundler
+
+Add following line to your Gemfile:
+
+```ruby
+gem "<%= gem_name %>"
+```
+
+And then execute:
+
+```
+$ bundle
+```
+
+## Configuration
+
+You can generate configuration template:
+
+```
+$ fluent-plugin-config-format <%= type %> <%= name %>
+```
+
+You can copy and paste generated documents here.
+
+## Copyright
+
+* Copyright(c) <%= Date.today.year %>- <%= user_name %>
+* License
+  * <%= @license.full_name %>