dtomasgu-fluentd 1.14.7.pre.dev

data/lib/fluent/command/ctl.rb

@@ -0,0 +1,180 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'optparse'
+require 'fluent/env'
+require 'fluent/version'
+if Fluent.windows?
+  require 'win32/event'
+  require 'win32/service'
+end
+
+module Fluent
+  class Ctl
+    DEFAULT_OPTIONS = {}
+
+    if Fluent.windows?
+      include Windows::ServiceConstants
+      include Windows::ServiceStructs
+      include Windows::ServiceFunctions
+
+      COMMAND_MAP = {
+        shutdown: "",
+        restart: "HUP",
+        flush: "USR1",
+        reload: "USR2",
+        dump: "CONT",
+      }
+      WINSVC_CONTROL_CODE_MAP = {
+        shutdown: SERVICE_CONTROL_STOP,
+        # 128 - 255: user-defined control code
+        # See https://docs.microsoft.com/en-us/windows/win32/api/winsvc/nf-winsvc-controlservice
+        restart: 128,
+        flush: 129,
+        reload: SERVICE_CONTROL_PARAMCHANGE,
+        dump: 130,
+      }
+    else
+      COMMAND_MAP = {
+        shutdown: :TERM,
+        restart: :HUP,
+        flush: :USR1,
+        reload: :USR2,
+        dump: :CONT,
+      }
+    end
+
+    def initialize(argv = ARGV)
+      @argv = argv
+      @options = {}
+      @opt_parser = OptionParser.new
+      configure_option_parser
+      @options.merge!(DEFAULT_OPTIONS)
+      parse_options!
+    end
+
+    def help_text
+      text = "\n"
+      if Fluent.windows?
+        text << "Usage: #{$PROGRAM_NAME} COMMAND [PID_OR_SVCNAME]\n"
+      else
+        text << "Usage: #{$PROGRAM_NAME} COMMAND PID\n"
+      end
+      text << "\n"
+      text << "Commands: \n"
+      COMMAND_MAP.each do |key, value|
+        text << "  #{key}\n"
+      end
+      text
+    end
+
+    def usage(msg = nil)
+      puts help_text
+      if msg
+        puts
+        puts "Error: #{msg}"
+      end
+      exit 1
+    end
+
+    def call
+      if Fluent.windows?
+        if @pid_or_svcname =~ /^[0-9]+$/
+          # Use as PID
+          return call_windows_event(@command, "fluentd_#{@pid_or_svcname}")
+        end
+
+        unless call_winsvc_control_code(@command, @pid_or_svcname)
+          puts "Cannot send control code to #{@pid_or_svcname} service, try to send an event with this name ..."
+          call_windows_event(@command, @pid_or_svcname)
+        end
+      else
+        call_signal(@command, @pid_or_svcname)
+      end
+    end
+
+    private
+
+    def call_signal(command, pid)
+      signal = COMMAND_MAP[command.to_sym]
+      Process.kill(signal, pid.to_i)
+    end
+
+    def call_winsvc_control_code(command, pid_or_svcname)
+      status = SERVICE_STATUS.new
+
+      begin
+        handle_scm = OpenSCManager(nil, nil, SC_MANAGER_CONNECT)
+        FFI.raise_windows_error('OpenSCManager') if handle_scm == 0
+
+        handle_scs = OpenService(handle_scm, "fluentdwinsvc", SERVICE_STOP | SERVICE_PAUSE_CONTINUE | SERVICE_USER_DEFINED_CONTROL)
+        FFI.raise_windows_error('OpenService') if handle_scs == 0
+
+        control_code = WINSVC_CONTROL_CODE_MAP[command.to_sym]
+
+        unless ControlService(handle_scs, control_code, status)
+          FFI.raise_windows_error('ControlService')
+        end
+      rescue => e
+        puts e
+        state = status[:dwCurrentState]
+        return state == SERVICE_STOPPED || state == SERVICE_STOP_PENDING
+      ensure
+        CloseServiceHandle(handle_scs)
+        CloseServiceHandle(handle_scm)
+      end
+
+      return true
+    end
+
+    def call_windows_event(command, pid_or_svcname)
+      prefix = pid_or_svcname
+      event_name = COMMAND_MAP[command.to_sym]
+      suffix = event_name.empty? ? "" : "_#{event_name}"
+
+      begin
+        event = Win32::Event.open("#{prefix}#{suffix}")
+        event.set
+        event.close
+      rescue Errno::ENOENT
+        puts "Error: Cannot find the fluentd process with the event name: \"#{prefix}\""
+      end
+    end
+
+    def configure_option_parser
+      @opt_parser.banner = help_text
+      @opt_parser.version = Fluent::VERSION
+    end
+
+    def parse_options!
+      @opt_parser.parse!(@argv)
+
+      @command = @argv[0]
+      @pid_or_svcname = @argv[1] || "fluentdwinsvc"
+
+      usage("Command isn't specified!") if @command.nil? || @command.empty?
+      usage("Unknown command: #{@command}") unless COMMAND_MAP.has_key?(@command.to_sym)
+
+      if Fluent.windows?
+        usage("PID or SVCNAME isn't specified!") if @pid_or_svcname.nil? || @pid_or_svcname.empty?
+      else
+        usage("PID isn't specified!") if @pid_or_svcname.nil? || @pid_or_svcname.empty?
+        usage("Invalid PID: #{pid}") unless @pid_or_svcname =~ /^[0-9]+$/
+      end
+    end
+  end
+end
+

data/lib/fluent/command/debug.rb

@@ -0,0 +1,103 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'optparse'
+
+op = OptionParser.new
+
+host = '127.0.0.1'
+port = 24230
+unix = nil
+
+op.on('-h', '--host HOST', "fluent host (default: #{host})") {|s|
+  host = s
+}
+
+op.on('-p', '--port PORT', "debug_agent tcp port (default: #{port})", Integer) {|i|
+  port = i
+}
+
+op.on('-u', '--unix PATH', "use unix socket instead of tcp") {|b|
+  unix = b
+}
+
+singleton_class.module_eval do
+  define_method(:usage) do |msg|
+    puts op.to_s
+    puts "error: #{msg}" if msg
+    exit 1
+  end
+end
+
+begin
+  op.parse!(ARGV)
+
+  if ARGV.length != 0
+    usage nil
+  end
+rescue
+  usage $!.to_s
+end
+
+require 'drb/drb'
+
+if unix
+  uri = "drbunix:#{unix}"
+else
+  uri = "druby://#{host}:#{port}"
+end
+
+require 'fluent/log'
+require 'fluent/env'
+require 'fluent/engine'
+require 'fluent/system_config'
+require 'serverengine'
+
+include Fluent::SystemConfig::Mixin
+
+dl_opts = {}
+dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE
+logger = ServerEngine::DaemonLogger.new(STDERR, dl_opts)
+$log = Fluent::Log.new(logger)
+Fluent::Engine.init(system_config)
+
+DRb::DRbObject.class_eval do
+  undef_method :methods
+  undef_method :instance_eval
+  undef_method :instance_variables
+  undef_method :instance_variable_get
+end
+
+remote_engine = DRb::DRbObject.new_with_uri(uri)
+
+Fluent.module_eval do
+  remove_const(:Engine)
+  const_set(:Engine, remote_engine)
+end
+
+include Fluent
+
+puts "Connected to #{uri}."
+puts "Usage:"
+puts "    Fluent::Engine.root_agent.event_router.match('some.tag') : get an output plugin instance"
+puts "    Fluent::Engine.root_agent.inputs[i]                      : get input plugin instances"
+puts "    Fluent::Plugin::OUTPUT_REGISTRY.lookup(name)             : load output plugin class (use this if you get DRb::DRbUnknown)"
+puts ""
+
+Encoding.default_internal = nil if Encoding.respond_to?(:default_internal)
+
+require 'irb'
+IRB.start

data/lib/fluent/command/fluentd.rb

@@ -0,0 +1,388 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'optparse'
+
+require 'fluent/supervisor'
+require 'fluent/log'
+require 'fluent/env'
+require 'fluent/version'
+
+$fluentdargv = Marshal.load(Marshal.dump(ARGV))
+
+op = OptionParser.new
+op.version = Fluent::VERSION
+
+opts = Fluent::Supervisor.default_options
+
+op.on('-s', "--setup [DIR=#{File.dirname(Fluent::DEFAULT_CONFIG_PATH)}]", "install sample configuration file to the directory") {|s|
+  opts[:setup_path] = s || File.dirname(Fluent::DEFAULT_CONFIG_PATH)
+}
+
+op.on('-c', '--config PATH', "config file path (default: #{Fluent::DEFAULT_CONFIG_PATH})") {|s|
+  opts[:config_path] = s
+}
+
+op.on('--dry-run', "Check fluentd setup is correct or not", TrueClass) {|b|
+  opts[:dry_run] = b
+}
+
+op.on('--show-plugin-config=PLUGIN', "[DEPRECATED] Show PLUGIN configuration and exit(ex: input:dummy)") {|plugin|
+  opts[:show_plugin_config] = plugin
+}
+
+op.on('-p', '--plugin DIR', "add plugin directory") {|s|
+  opts[:plugin_dirs] << s
+}
+
+op.on('-I PATH', "add library path") {|s|
+  $LOAD_PATH << s
+}
+
+op.on('-r NAME', "load library") {|s|
+  opts[:libs] << s
+}
+
+op.on('-d', '--daemon PIDFILE', "daemonize fluent process") {|s|
+  opts[:daemonize] = s
+}
+
+op.on('--under-supervisor', "run fluent worker under supervisor (this option is NOT for users)") {
+  opts[:supervise] = false
+}
+
+op.on('--no-supervisor', "run fluent worker without supervisor") {
+  opts[:supervise] = false
+  opts[:standalone_worker] = true
+}
+
+op.on('--workers NUM', "specify the number of workers under supervisor") { |i|
+  opts[:workers] = i.to_i
+}
+
+op.on('--user USER', "change user") {|s|
+  opts[:chuser] = s
+}
+
+op.on('--group GROUP', "change group") {|s|
+  opts[:chgroup] = s
+}
+
+op.on('--umask UMASK', "change umask") {|s|
+  opts[:chumask] = s
+}
+
+op.on('-o', '--log PATH', "log file path") {|s|
+  opts[:log_path] = s
+}
+
+op.on('--log-rotate-age AGE', 'generations to keep rotated log files') {|age|
+  if Fluent::Log::LOG_ROTATE_AGE.include?(age)
+    opts[:log_rotate_age] = age
+  else
+    begin
+      opts[:log_rotate_age] = Integer(age)
+    rescue TypeError, ArgumentError
+      usage "log-rotate-age should be #{ROTATE_AGE.join(', ')} or a number"
+    end
+  end
+}
+
+op.on('--log-rotate-size BYTES', 'sets the byte size to rotate log files') {|s|
+  opts[:log_rotate_size] = s.to_i
+}
+
+op.on('--log-event-verbose', 'enable log events during process startup/shutdown') {|b|
+  opts[:log_event_verbose] = b
+}
+
+op.on('-i', '--inline-config CONFIG_STRING', "inline config which is appended to the config file on-the-fly") {|s|
+  opts[:inline_config] = s
+}
+
+op.on('--emit-error-log-interval SECONDS', "suppress interval seconds of emit error logs") {|s|
+  opts[:suppress_interval] = s.to_i
+}
+
+op.on('--suppress-repeated-stacktrace [VALUE]', "suppress repeated stacktrace", TrueClass) {|b|
+  b = true if b.nil?
+  opts[:suppress_repeated_stacktrace] = b
+}
+
+op.on('--without-source', "invoke a fluentd without input plugins", TrueClass) {|b|
+  opts[:without_source] = b
+}
+
+op.on('--config-file-type VALU', 'guessing file type of fluentd configuration. yaml/yml or guess') { |s|
+  if (s == 'yaml') || (s == 'yml')
+    opts[:config_file_type] = s.to_sym
+  elsif (s == 'guess')
+    opts[:config_file_type] = s.to_sym
+  else
+    usage '--config-file-type accepts yaml/yml or guess'
+  end
+}
+
+op.on('--use-v1-config', "Use v1 configuration format (default)", TrueClass) {|b|
+  opts[:use_v1_config] = b
+}
+
+op.on('--use-v0-config', "Use v0 configuration format", TrueClass) {|b|
+  opts[:use_v1_config] = !b
+}
+
+op.on('--strict-config-value', "Parse config values strictly", TrueClass) {|b|
+  opts[:strict_config_value] = b
+}
+
+op.on('--enable-input-metrics', "Enable input plugin metrics on fluentd", TrueClass) {|b|
+  opts[:enable_input_metrics] = b
+}
+
+op.on('--enable-size-metrics', "Enable plugin record size metrics on fluentd", TrueClass) {|b|
+  opts[:enable_size_metrics] = b
+}
+
+op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
+  if b
+    opts[:log_level] = [opts[:log_level] - 1, Fluent::Log::LEVEL_TRACE].max
+  end
+}
+
+op.on('-q', '--quiet', "decrease verbose level (-q: warn, -qq: error)", TrueClass) {|b|
+  if b
+    opts[:log_level] = [opts[:log_level] + 1, Fluent::Log::LEVEL_ERROR].min
+  end
+}
+
+op.on('--suppress-config-dump', "suppress config dumping when fluentd starts", TrueClass) {|b|
+  opts[:suppress_config_dump] = b
+}
+
+op.on('-g', '--gemfile GEMFILE', "Gemfile path") {|s|
+  opts[:gemfile] = s
+}
+
+op.on('-G', '--gem-path GEM_INSTALL_PATH', "Gemfile install path (default: $(dirname $gemfile)/vendor/bundle)") {|s|
+  opts[:gem_install_path] = s
+}
+
+op.on('--conf-encoding ENCODING', "specify configuration file encoding") { |s|
+  opts[:conf_encoding] = s
+}
+
+op.on('--disable-shared-socket', "Don't open shared socket for multiple workers") { |b|
+  opts[:disable_shared_socket] = b
+}
+
+if Fluent.windows?
+  require 'windows/library'
+  include Windows::Library
+
+  opts.merge!(
+    :winsvc_name => 'fluentdwinsvc',
+    :winsvc_display_name => 'Fluentd Windows Service',
+    :winsvc_desc => 'Fluentd is an event collector system.',
+  )
+
+  op.on('-x', '--signame INTSIGNAME', "an object name which is used for Windows Service signal (Windows only)") {|s|
+    opts[:signame] = s
+  }
+
+  op.on('--reg-winsvc MODE', "install/uninstall as Windows Service. (i: install, u: uninstall) (Windows only)") {|s|
+    opts[:regwinsvc] = s
+  }
+
+  op.on('--[no-]reg-winsvc-auto-start', "Automatically start the Windows Service at boot. (only effective with '--reg-winsvc i') (Windows only)") {|s|
+    opts[:regwinsvcautostart] = s
+  }
+
+  op.on('--[no-]reg-winsvc-delay-start', "Automatically start the Windows Service at boot with delay. (only effective with '--reg-winsvc i' and '--reg-winsvc-auto-start') (Windows only)") {|s|
+    opts[:regwinsvcdelaystart] = s
+  }
+
+  op.on('--reg-winsvc-fluentdopt OPTION', "specify fluentd option parameters for Windows Service. (Windows only)") {|s|
+    opts[:fluentdopt] = s
+  }
+
+  op.on('--winsvc-name NAME', "The Windows Service name to run as (Windows only)") {|s|
+    opts[:winsvc_name] = s
+  }
+
+  op.on('--winsvc-display-name DISPLAY_NAME', "The Windows Service display name (Windows only)") {|s|
+    opts[:winsvc_display_name] = s
+  }
+
+  op.on('--winsvc-desc DESC', "The Windows Service description (Windows only)") {|s|
+    opts[:winsvc_desc] = s
+  }
+end
+
+
+singleton_class.module_eval do
+  define_method(:usage) do |msg|
+    puts op.to_s
+    puts "error: #{msg}" if msg
+    exit 1
+  end
+end
+
+begin
+  rest = op.parse(ARGV)
+
+  if rest.length != 0
+    usage nil
+  end
+rescue
+  usage $!.to_s
+end
+
+
+##
+## Bundler injection
+#
+if ENV['FLUENTD_DISABLE_BUNDLER_INJECTION'] != '1' && gemfile = opts[:gemfile]
+  ENV['BUNDLE_GEMFILE'] = gemfile
+  if path = opts[:gem_install_path]
+    ENV['BUNDLE_PATH'] = path
+  else
+    ENV['BUNDLE_PATH'] = File.expand_path(File.join(File.dirname(gemfile), 'vendor/bundle'))
+  end
+  ENV['FLUENTD_DISABLE_BUNDLER_INJECTION'] = '1'
+  load File.expand_path(File.join(File.dirname(__FILE__), 'bundler_injection.rb'))
+end
+
+if setup_path = opts[:setup_path]
+  require 'fileutils'
+  FileUtils.mkdir_p File.join(setup_path, "plugin")
+  confpath = File.join(setup_path, "fluent.conf")
+  if File.exist?(confpath)
+    puts "#{confpath} already exists."
+  else
+    File.open(confpath, "w") {|f|
+      conf = File.read File.join(File.dirname(__FILE__), "..", "..", "..", "fluent.conf")
+      f.write conf
+    }
+    puts "Installed #{confpath}."
+  end
+  exit 0
+end
+
+early_exit = false
+start_service = false
+if winsvcinstmode = opts[:regwinsvc]
+  require 'fileutils'
+  require "win32/service"
+  require "win32/registry"
+  include Win32
+
+  case winsvcinstmode
+  when 'i'
+    binary_path = File.join(File.dirname(__FILE__), "..")
+    ruby_path = "\0" * 256
+    GetModuleFileName.call(0,ruby_path,256)
+    ruby_path = ruby_path.rstrip.gsub(/\\/, '/')
+    start_type = Service::DEMAND_START
+    if opts[:regwinsvcautostart]
+      start_type = Service::AUTO_START
+      start_service = true
+    end
+
+
+    Service.create(
+      service_name: opts[:winsvc_name],
+      host: nil,
+      service_type: Service::WIN32_OWN_PROCESS,
+      description: opts[:winsvc_desc],
+      start_type: start_type,
+      error_control: Service::ERROR_NORMAL,
+      binary_path_name: "\"#{ruby_path}\" -C \"#{binary_path}\"  winsvc.rb --service-name #{opts[:winsvc_name]}",
+      load_order_group: "",
+      dependencies: [""],
+      display_name: opts[:winsvc_display_name]
+    )
+
+    if opts[:regwinsvcdelaystart]
+      Service.configure(
+        service_name: opts[:winsvc_name],
+        delayed_start: true
+      )
+    end
+  when 'u'
+    if Service.status(opts[:winsvc_name]).current_state != 'stopped'
+      begin
+        Service.stop(opts[:winsvc_name])
+      rescue => ex
+        puts "Warning: Failed to stop service: ", ex
+      end
+    end
+    Service.delete(opts[:winsvc_name])
+  else
+    # none
+  end
+  early_exit = true
+end
+
+if fluentdopt = opts[:fluentdopt]
+  Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\#{opts[:winsvc_name]}", Win32::Registry::KEY_ALL_ACCESS) do |reg|
+    reg['fluentdopt', Win32::Registry::REG_SZ] = fluentdopt
+  end
+  early_exit = true
+end
+
+if start_service
+  Service.start(opts[:winsvc_name])
+end
+
+exit 0 if early_exit
+
+if opts[:supervise]
+  if Fluent.windows?
+    if opts[:log_path] && opts[:log_path] != "-"
+      if opts[:log_rotate_age] || opts[:log_rotate_size]
+        require 'pathname'
+
+        log_path = Pathname(opts[:log_path]).sub_ext("-supervisor#{Pathname(opts[:log_path]).extname}").to_s
+        opts[:log_path] = log_path
+      end
+    end
+  end
+
+  supervisor = Fluent::Supervisor.new(opts)
+  supervisor.configure(supervisor: true)
+  supervisor.run_supervisor(dry_run: opts[:dry_run])
+else
+  if opts[:standalone_worker] && opts[:workers] && opts[:workers] > 1
+    puts "Error: multi workers is not supported with --no-supervisor"
+    exit 2
+  end
+  worker = Fluent::Supervisor.new(opts)
+  worker.configure
+
+  if opts[:daemonize] && opts[:standalone_worker]
+    require 'fluent/daemonizer'
+    args = ARGV.dup
+    i = args.index('--daemon')
+    args.delete_at(i + 1)          # value of --daemon
+    args.delete_at(i)              # --daemon itself
+
+    Fluent::Daemonizer.daemonize(opts[:daemonize], args) do
+      worker.run_worker
+    end
+  else
+    worker.run_worker
+  end
+end