fluentd 0.14.17-x64-mingw32 → 1.3.1-x64-mingw32

data/lib/fluent/supervisor.rb

@@ -17,6 +17,7 @@
 require 'fileutils'
 
 require 'fluent/config'
+require 'fluent/counter'
 require 'fluent/env'
 require 'fluent/engine'
 require 'fluent/error'
@@ -41,6 +42,8 @@ module Fluent
   module ServerModule
     def before_run
       @start_time = Time.now
+      @rpc_server = nil
+      @counter = nil
 
       if config[:rpc_endpoint]
         @rpc_endpoint = config[:rpc_endpoint]
@@ -54,6 +57,10 @@ module Fluent
         install_windows_event_handler
       end
 
+      if counter = config[:counter_server]
+        run_counter_server(counter)
+      end
+
       socket_manager_path = ServerEngine::SocketManager::Server.generate_path
       ServerEngine::SocketManager::Server.open(socket_manager_path)
       ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
@@ -61,6 +68,7 @@ module Fluent
 
     def after_run
       stop_rpc_server if @rpc_endpoint
+      stop_counter_server if @counter
       Fluent::Supervisor.cleanup_resources
     end
 
@@ -126,6 +134,18 @@ module Fluent
       @rpc_server.shutdown
     end
 
+    def run_counter_server(counter_conf)
+      @counter = Fluent::Counter::Server.new(
+        counter_conf.scope,
+        {host: counter_conf.bind, port: counter_conf.port, log: $log, path: counter_conf.backup_path}
+      )
+      @counter.start
+    end
+
+    def stop_counter_server
+      @counter.stop
+    end
+
     def install_supervisor_signal_handlers
       trap :HUP do
         $log.debug "fluentd supervisor process get SIGHUP"
@@ -160,21 +180,31 @@ module Fluent
 
     def supervisor_sigusr1_handler
       if log = config[:logger_initializer]
-        log.reopen!
+        # Creating new thread due to mutex can't lock
+        # in main thread during trap context
+        Thread.new {
+          log.reopen!
+        }.run
       end
 
-      if pid = config[:worker_pid]
-        Process.kill(:USR1, pid)
-        # don't rescue Erro::ESRSH here (invalid status)
+      if config[:worker_pid]
+        config[:worker_pid].each_value do |pid|
+          Process.kill(:USR1, pid)
+          # don't rescue Errno::ESRCH here (invalid status)
+        end
       end
     end
 
     def kill_worker
-      if pid = config[:worker_pid]
-        if Fluent.windows?
-          Process.kill :KILL, pid
-        else
-          Process.kill :TERM, pid
+      if config[:worker_pid]
+        pids = config[:worker_pid].clone
+        config[:worker_pid].clear
+        pids.each_value do |pid|
+          if Fluent.windows?
+            Process.kill :KILL, pid
+          else
+            Process.kill :TERM, pid
+          end
         end
       end
     end
@@ -198,7 +228,7 @@ module Fluent
     end
 
     def after_start
-      config[:worker_pid] = @pm.pid
+      (config[:worker_pid] ||= {})[@worker_id] = @pm.pid
     end
   end
 
@@ -218,7 +248,8 @@ module Fluent
 
       config_fname = File.basename(path)
       config_basedir = File.dirname(path)
-      config_data = File.read(path)
+      # Assume fluent.conf encoding is UTF-8
+      config_data = File.open(path, "r:utf-8:utf-8") {|f| f.read }
       inline_config = params['inline_config']
       if inline_config == '-'
         config_data << "\n" << STDIN.read
@@ -242,6 +273,7 @@ module Fluent
       log_rotate_size = params['log_rotate_size']
       rpc_endpoint = system_config.rpc_endpoint
       enable_get_dump = system_config.enable_get_dump
+      counter_server = system_config.counter_server
 
       log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace}
       logger_initializer = Supervisor::LoggerInitializer.new(
@@ -283,6 +315,7 @@ module Fluent
           suppress_repeated_stacktrace: suppress_repeated_stacktrace,
           daemonize: daemonize,
           rpc_endpoint: rpc_endpoint,
+          counter_server: counter_server,
           enable_get_dump: enable_get_dump,
           windows_daemon_cmdline: [ServerEngine.ruby_bin_path,
                                    File.join(File.dirname(__FILE__), 'daemon.rb'),
@@ -359,6 +392,11 @@ module Fluent
         self
       end
 
+      def apply_options(opts)
+        $log.format = opts[:format] if opts[:format]
+        $log.time_format = opts[:time_format] if opts[:time_format]
+      end
+
       def level=(level)
         @level = level
         $log.level = level
@@ -438,6 +476,9 @@ module Fluent
       show_plugin_config if @show_plugin_config
       read_config
       set_system_config
+      @log.apply_options(format: @system_config.log.format, time_format: @system_config.log.time_format)
+
+      $log.info :supervisor, "parsing config file is succeeded", path: @config_path
 
       if @workers < 1
         raise Fluent::ConfigError, "invalid number of workers (must be > 0):#{@workers}"
@@ -484,11 +525,12 @@ module Fluent
                      else :workers
                      end
       @log.init(process_type, worker_id)
-      Process.setproctitle("worker:#{@process_name}") if @process_name
-
       show_plugin_config if @show_plugin_config
       read_config
       set_system_config
+      @log.apply_options(format: @system_config.log.format, time_format: @system_config.log.time_format)
+
+      Process.setproctitle("worker:#{@process_name}") if @process_name
 
       if @standalone_worker && @workers != 1
         raise Fluent::ConfigError, "invalid number of workers (must be 1 or unspecified) with --no-supervisor: #{@workers}"
@@ -497,11 +539,11 @@ module Fluent
       install_main_process_signal_handlers
 
       # This is the only log messsage for @standalone_worker
-      $log.info "starting fluentd-#{Fluent::VERSION} without supervision", pid: Process.pid if @standalone_worker
+      $log.info "starting fluentd-#{Fluent::VERSION} without supervision", pid: Process.pid, ruby: RUBY_VERSION if @standalone_worker
 
       main_process do
         create_socket_manager if @standalone_worker
-        change_privilege
+        change_privilege if @standalone_worker
         init_engine
         run_configure
         run_engine
@@ -519,7 +561,7 @@ module Fluent
     end
 
     def dry_run_cmd
-      $log.info "starting fluentd-#{Fluent::VERSION} as dry run mode"
+      $log.info "starting fluentd-#{Fluent::VERSION} as dry run mode", ruby: RUBY_VERSION
       @system_config.suppress_config_dump = true
       dry_run
       exit 0
@@ -555,7 +597,7 @@ module Fluent
       dry_run
 
       Process.setproctitle("supervisor:#{@process_name}") if @process_name
-      $log.info "starting fluentd-#{Fluent::VERSION}", pid: Process.pid
+      $log.info "starting fluentd-#{Fluent::VERSION}", pid: Process.pid, ruby: RUBY_VERSION
 
       rubyopt = ENV["RUBYOPT"]
       fluentd_spawn_cmd = [ServerEngine.ruby_bin_path, "-Eascii-8bit:ascii-8bit"]
@@ -647,14 +689,13 @@ module Fluent
     end
 
     def flush_buffer
-      $log.debug "fluentd main process get SIGUSR1"
-      $log.info "force flushing buffered events"
-      @log.reopen!
-
       # Creating new thread due to mutex can't lock
       # in main thread during trap context
       Thread.new {
         begin
+          $log.debug "fluentd main process get SIGUSR1"
+          $log.info "force flushing buffered events"
+          @log.reopen!
           Fluent::Engine.flush!
           $log.debug "flushing thread: flushed"
         rescue Exception => e
@@ -714,10 +755,9 @@ module Fluent
     end
 
     def read_config
-      $log.info :supervisor, "reading config file", path: @config_path
       @config_fname = File.basename(@config_path)
       @config_basedir = File.dirname(@config_path)
-      @config_data = File.read(@config_path)
+      @config_data = File.open(@config_path, "r:utf-8:utf-8") {|f| f.read }
       if @inline_config == '-'
         @config_data << "\n" << STDIN.read
       elsif @inline_config

data/lib/fluent/system_config.rb

@@ -26,7 +26,7 @@ module Fluent
       :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
       :log_event_verbose,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
-      :file_permission, :dir_permission,
+      :file_permission, :dir_permission, :counter_server, :counter_client,
     ]
 
     config_param :workers,   :integer, default: 1
@@ -46,6 +46,32 @@ module Fluent
     config_param :dir_permission, default: nil do |v|
       v.to_i(8)
     end
+    config_section :log, required: false, init: true, multi: false do
+      config_param :format, :enum, list: [:text, :json], default: :text
+      config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z'
+    end
+
+    config_section :counter_server, multi: false do
+      desc 'scope name of counter server'
+      config_param :scope, :string
+
+      desc 'the port of counter server to listen to'
+      config_param :port, :integer, default: nil
+      desc 'the bind address of counter server to listen to'
+      config_param :bind, :string, default: nil
+
+      desc 'backup file path of counter values'
+      config_param :backup_path, :string
+    end
+
+    config_section :counter_client, multi: false do
+      desc 'the port of counter server'
+      config_param :port, :integer, default: nil
+      desc 'the IP address or hostname of counter server'
+      config_param :host, :string
+      desc 'the timeout of each operation'
+      config_param :timeout, :time, default: nil
+    end
 
     def self.create(conf)
       systems = conf.elements(name: 'system')
@@ -94,7 +120,7 @@ module Fluent
       supervisor.instance_eval {
         SYSTEM_CONFIG_PARAMETERS.each do |param|
           case param
-          when :rpc_endpoint, :enable_get_dump, :process_name, :file_permission, :dir_permission
+          when :rpc_endpoint, :enable_get_dump, :process_name, :file_permission, :dir_permission, :counter_server, :counter_client
             next # doesn't exist in command line options
           when :emit_error_log_interval
             system.emit_error_log_interval = @suppress_interval if @suppress_interval
@@ -132,6 +158,8 @@ module Fluent
             instance_variable_set("@#{param}", param_value)
           end
         end
+        #@counter_server = system.counter_server unless system.counter_server.nil?
+        #@counter_client = system.counter_client unless system.counter_client.nil?
       }
     end
 

data/lib/fluent/test/helpers.rb

@@ -21,7 +21,7 @@ require 'fluent/time'
 module Fluent
   module Test
     module Helpers
-      # See "Example Custom Assertion: http://test-unit.github.io/test-unit/en/Test/Unit/Assertions.html
+      # See "Example Custom Assertion: https://test-unit.github.io/test-unit/en/Test/Unit/Assertions.html
       def assert_equal_event_time(expected, actual, message = nil)
         expected_s = "#{Time.at(expected.sec)} (nsec #{expected.nsec})"
         actual_s   = "#{Time.at(actual.sec)  } (nsec #{actual.nsec})"

data/lib/fluent/time.rb

@@ -24,6 +24,7 @@ require 'fluent/config/error'
 module Fluent
   class EventTime
     TYPE = 0
+    FORMATTER = Strftime.new('%Y-%m-%d %H:%M:%S.%N %z')
 
     def initialize(sec, nsec = 0)
       @sec = sec
@@ -108,6 +109,10 @@ module Fluent
     def method_missing(name, *args, &block)
       @sec.send(name, *args, &block)
     end
+
+    def inspect
+      FORMATTER.exec(Time.at(self))
+    end
   end
 
   module TimeMixin
@@ -319,21 +324,24 @@ module Fluent
       @tc2 = 0
       @tc2_str = nil
 
+      strftime = format && (Strftime.new(format) rescue nil)
       if format && format =~ /(^|[^%])(%%)*%L|(^|[^%])(%%)*%\d*N/
         define_singleton_method(:format, method(:format_with_subsec))
         define_singleton_method(:call, method(:format_with_subsec))
       else
         define_singleton_method(:format, method(:format_without_subsec))
-        define_singleton_method(:call, method(:format_with_subsec))
+        define_singleton_method(:call, method(:format_without_subsec))
       end
 
-      formatter = Fluent::Timezone.formatter(timezone, format)
+      formatter = Fluent::Timezone.formatter(timezone, strftime ? strftime : format)
       @format_nocache = case
-                        when formatter           then formatter
-                        when format && localtime then ->(time){ Time.at(time).strftime(format) }
-                        when format              then ->(time){ Time.at(time).utc.strftime(format) }
-                        when localtime           then ->(time){ Time.at(time).iso8601 }
-                        else                          ->(time){ Time.at(time).utc.iso8601 }
+                        when formatter             then formatter
+                        when strftime && localtime then ->(time){ strftime.exec(Time.at(time)) }
+                        when format && localtime   then ->(time){ Time.at(time).strftime(format) }
+                        when strftime              then ->(time){ strftime.exec(Time.at(time).utc) }
+                        when format                then ->(time){ Time.at(time).utc.strftime(format) }
+                        when localtime             then ->(time){ Time.at(time).iso8601 }
+                        else                            ->(time){ Time.at(time).utc.iso8601 }
                         end
     end
 

data/lib/fluent/timezone.rb

@@ -97,10 +97,15 @@ module Fluent
       if NUMERIC_PATTERN === timezone
         offset = Time.zone_offset(timezone)
 
-        if format
+        case
+        when format.is_a?(String)
           return Proc.new {|time|
             Time.at(time).localtime(offset).strftime(format)
           }
+        when format.is_a?(Strftime)
+          return Proc.new {|time|
+            format.exec(Time.at(time).localtime(offset))
+          }
         else
           return Proc.new {|time|
             Time.at(time).localtime(offset).iso8601
@@ -116,10 +121,15 @@ module Fluent
           return nil
         end
 
-        if format
+        case
+        when format.is_a?(String)
           return Proc.new {|time|
             Time.at(time).localtime(tz.period_for_utc(time).utc_total_offset).strftime(format)
           }
+        when format.is_a?(Strftime)
+          return Proc.new {|time|
+            format.exec(Time.at(time).localtime(tz.period_for_utc(time).utc_total_offset))
+          }
         else
           return Proc.new {|time|
             Time.at(time).localtime(tz.period_for_utc(time).utc_total_offset).iso8601
@@ -129,5 +139,19 @@ module Fluent
 
       return nil
     end
+
+    def self.utc_offset(timezone)
+      return 0 if timezone.nil?
+
+      case timezone
+      when NUMERIC_PATTERN
+        Time.zone_offset(timezone)
+      when NAME_PATTERN
+        tz = TZInfo::Timezone.get(timezone)
+        ->(time) {
+          tz.period_for_utc(time).utc_total_offset
+        }
+      end
+    end
   end
 end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.17'
+  VERSION = '1.3.1'
 
 end

data/templates/new_gem/README.md.erb

@@ -1,6 +1,6 @@
 # <%= gem_name %>
 
-[Fluentd](http://fluentd.org/) <%= type %> plugin to do something.
+[Fluentd](https://fluentd.org/) <%= type %> plugin to do something.
 
 TODO: write description for you plugin.
 
@@ -31,7 +31,7 @@ $ bundle
 You can generate configuration template:
 
 ```
-$ fluent-plugin-format-config <%= type %> <%= name %>
+$ fluent-plugin-config-format <%= type %> <%= name %>
 ```
 
 You can copy and paste generated documents here.

data/templates/new_gem/lib/fluent/plugin/filter.rb.erb

@@ -5,7 +5,7 @@ require "fluent/plugin/filter"
 module Fluent
   module Plugin
     class <%= class_name %> < Fluent::Plugin::Filter
-      Fluent::Plugin.register_filter("<%= name %>", self)
+      Fluent::Plugin.register_filter("<%= plugin_name %>", self)
 
       def filter(tag, time, record)
       end

data/templates/new_gem/lib/fluent/plugin/input.rb.erb

@@ -5,7 +5,7 @@ require "fluent/plugin/input"
 module Fluent
   module Plugin
     class <%= class_name %> < Fluent::Plugin::Input
-      Fluent::Plugin.register_input("<%= name %>", self)
+      Fluent::Plugin.register_input("<%= plugin_name %>", self)
     end
   end
 end

data/templates/new_gem/lib/fluent/plugin/output.rb.erb

@@ -5,7 +5,7 @@ require "fluent/plugin/output"
 module Fluent
   module Plugin
     class <%= class_name %> < Fluent::Plugin::Output
-      Fluent::Plugin.register_output("<%= name %>", self)
+      Fluent::Plugin.register_output("<%= plugin_name %>", self)
     end
   end
 end