fluentd-hubspot 0.14.14.1

data/lib/fluent/command/fluentd.rb

@@ -0,0 +1,301 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'optparse'
+
+require 'fluent/supervisor'
+require 'fluent/log'
+require 'fluent/env'
+require 'fluent/version'
+
+$fluentdargv = Marshal.load(Marshal.dump(ARGV))
+
+op = OptionParser.new
+op.version = Fluent::VERSION
+
+opts = Fluent::Supervisor.default_options
+
+op.on('-s', "--setup [DIR=#{File.dirname(Fluent::DEFAULT_CONFIG_PATH)}]", "install sample configuration file to the directory") {|s|
+  opts[:setup_path] = s || File.dirname(Fluent::DEFAULT_CONFIG_PATH)
+}
+
+op.on('-c', '--config PATH', "config file path (default: #{Fluent::DEFAULT_CONFIG_PATH})") {|s|
+  opts[:config_path] = s
+}
+
+op.on('--dry-run', "Check fluentd setup is correct or not", TrueClass) {|b|
+  opts[:dry_run] = b
+}
+
+op.on('--show-plugin-config=PLUGIN', "Show PLUGIN configuration and exit(ex: input:dummy)") {|plugin|
+  opts[:show_plugin_config] = plugin
+}
+
+op.on('-p', '--plugin DIR', "add plugin directory") {|s|
+  opts[:plugin_dirs] << s
+}
+
+op.on('-I PATH', "add library path") {|s|
+  $LOAD_PATH << s
+}
+
+op.on('-r NAME', "load library") {|s|
+  opts[:libs] << s
+}
+
+op.on('-d', '--daemon PIDFILE', "daemonize fluent process") {|s|
+  opts[:daemonize] = s
+}
+
+op.on('--under-supervisor', "run fluent worker under supervisor (this option is NOT for users)") {
+  opts[:supervise] = false
+}
+
+op.on('--no-supervisor', "run fluent worker without supervisor") {
+  opts[:supervise] = false
+  opts[:standalone_worker] = true
+}
+
+op.on('--workers NUM', "specify the number of workers under supervisor") { |i|
+  opts[:workers] = i.to_i
+}
+
+op.on('--user USER', "change user") {|s|
+  opts[:chuser] = s
+}
+
+op.on('--group GROUP', "change group") {|s|
+  opts[:chgroup] = s
+}
+
+op.on('-o', '--log PATH', "log file path") {|s|
+  opts[:log_path] = s
+}
+
+ROTATE_AGE = %w(daily weekly monthly)
+op.on('--log-rotate-age AGE', 'generations to keep rotated log files') {|age|
+  if ROTATE_AGE.include?(age)
+    opts[:log_rotate_age] = age
+  else
+    begin
+      opts[:log_rotate_age] = Integer(age)
+    rescue TypeError
+      usage "log-rotate-age should be #{rotate_ages.join(', ')} or a number"
+    end
+  end
+}
+
+op.on('--log-rotate-size BYTES', 'sets the byte size to rotate log files') {|s|
+  opts[:log_rotate_size] = s.to_i
+}
+
+op.on('--log-event-verbose', 'enable log events during process startup/shutdown') {|b|
+  opts[:log_event_verbose] = b
+}
+
+op.on('-i', '--inline-config CONFIG_STRING', "inline config which is appended to the config file on-the-fly") {|s|
+  opts[:inline_config] = s
+}
+
+op.on('--emit-error-log-interval SECONDS', "suppress interval seconds of emit error logs") {|s|
+  opts[:suppress_interval] = s.to_i
+}
+
+op.on('--suppress-repeated-stacktrace [VALUE]', "suppress repeated stacktrace", TrueClass) {|b|
+  b = true if b.nil?
+  opts[:suppress_repeated_stacktrace] = b
+}
+
+op.on('--without-source', "invoke a fluentd without input plugins", TrueClass) {|b|
+  opts[:without_source] = b
+}
+
+op.on('--use-v1-config', "Use v1 configuration format (default)", TrueClass) {|b|
+  opts[:use_v1_config] = b
+}
+
+op.on('--use-v0-config', "Use v0 configuration format", TrueClass) {|b|
+  opts[:use_v1_config] = !b
+}
+
+op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
+  if b
+    opts[:log_level] = [opts[:log_level] - 1, Fluent::Log::LEVEL_TRACE].max
+  end
+}
+
+op.on('-q', '--quiet', "decrease verbose level (-q: warn, -qq: error)", TrueClass) {|b|
+  if b
+    opts[:log_level] = [opts[:log_level] + 1, Fluent::Log::LEVEL_ERROR].min
+  end
+}
+
+op.on('--suppress-config-dump', "suppress config dumping when fluentd starts", TrueClass) {|b|
+  opts[:suppress_config_dump] = b
+}
+
+op.on('-g', '--gemfile GEMFILE', "Gemfile path") {|s|
+  opts[:gemfile] = s
+}
+
+op.on('-G', '--gem-path GEM_INSTALL_PATH', "Gemfile install path (default: $(dirname $gemfile)/vendor/bundle)") {|s|
+  opts[:gem_install_path] = s
+}
+
+if Fluent.windows?
+  require 'windows/library'
+  include Windows::Library
+
+  op.on('-x', '--signame INTSIGNAME', "an object name which is used for Windows Service signal (Windows only)") {|s|
+    opts[:signame] = s
+  }
+
+  op.on('--reg-winsvc MODE', "install/uninstall as Windows Service. (i: install, u: uninstall) (Windows only)") {|s|
+    opts[:regwinsvc] = s
+  }
+
+  op.on('--[no-]reg-winsvc-auto-start', "Automatically start the Windows Service at boot. (only effective with '--reg-winsvc i') (Windows only)") {|s|
+    opts[:regwinsvcautostart] = s
+  }
+
+  op.on('--reg-winsvc-fluentdopt OPTION', "specify fluentd option paramters for Windows Service. (Windows only)") {|s|
+    opts[:fluentdopt] = s
+  }
+end
+
+
+(class << self; self; end).module_eval do
+  define_method(:usage) do |msg|
+    puts op.to_s
+    puts "error: #{msg}" if msg
+    exit 1
+  end
+end
+
+begin
+  rest = op.parse(ARGV)
+
+  if rest.length != 0
+    usage nil
+  end
+rescue
+  usage $!.to_s
+end
+
+
+##
+## Bundler injection
+#
+if ENV['FLUENTD_DISABLE_BUNDLER_INJECTION'] != '1' && gemfile = opts[:gemfile]
+  ENV['BUNDLE_GEMFILE'] = gemfile
+  if path = opts[:gem_install_path]
+    ENV['BUNDLE_PATH'] = path
+  else
+    ENV['BUNDLE_PATH'] = File.expand_path(File.join(File.dirname(gemfile), 'vendor/bundle'))
+  end
+  ENV['FLUENTD_DISABLE_BUNDLER_INJECTION'] = '1'
+  load File.expand_path(File.join(File.dirname(__FILE__), 'bundler_injection.rb'))
+end
+
+if setup_path = opts[:setup_path]
+  require 'fileutils'
+  FileUtils.mkdir_p File.join(setup_path, "plugin")
+  confpath = File.join(setup_path, "fluent.conf")
+  if File.exist?(confpath)
+    puts "#{confpath} already exists."
+  else
+    File.open(confpath, "w") {|f|
+      conf = File.read File.join(File.dirname(__FILE__), "..", "..", "..", "fluent.conf")
+      f.write conf
+    }
+    puts "Installed #{confpath}."
+  end
+  exit 0
+end
+
+early_exit = false
+start_service = false
+if winsvcinstmode = opts[:regwinsvc]
+  FLUENTD_WINSVC_NAME="fluentdwinsvc"
+  FLUENTD_WINSVC_DISPLAYNAME="Fluentd Windows Service"
+  FLUENTD_WINSVC_DESC="Fluentd is an event collector system."
+  require 'fileutils'
+  require "win32/service"
+  require "win32/registry"
+  include Win32
+
+  case winsvcinstmode
+  when 'i'
+    binary_path = File.join(File.dirname(__FILE__), "..")
+    ruby_path = "\0" * 256
+    GetModuleFileName.call(0,ruby_path,256)
+    ruby_path = ruby_path.rstrip.gsub(/\\/, '/')
+    start_type = Service::DEMAND_START
+    if opts[:regwinsvcautostart]
+      start_type = Service::AUTO_START
+      start_service = true
+    end
+
+    Service.create(
+      service_name: FLUENTD_WINSVC_NAME,
+      host: nil,
+      service_type: Service::WIN32_OWN_PROCESS,
+      description: FLUENTD_WINSVC_DESC,
+      start_type: start_type,
+      error_control: Service::ERROR_NORMAL,
+      binary_path_name: ruby_path+" -C "+binary_path+" winsvc.rb",
+      load_order_group: "",
+      dependencies: [""],
+      display_name: FLUENTD_WINSVC_DISPLAYNAME
+    )
+  when 'u'
+    if Service.status(FLUENTD_WINSVC_NAME).current_state != 'stopped'
+      begin
+        Service.stop(FLUENTD_WINSVC_NAME)
+      rescue => ex
+        puts "Warning: Failed to stop service: ", ex
+      end
+    end
+    Service.delete(FLUENTD_WINSVC_NAME)
+  else
+    # none
+  end
+  early_exit = true
+end
+
+if fluentdopt = opts[:fluentdopt]
+  Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\fluentdwinsvc", Win32::Registry::KEY_ALL_ACCESS) do |reg|
+    reg['fluentdopt', Win32::Registry::REG_SZ] = fluentdopt
+  end
+  early_exit = true
+end
+
+if start_service
+  Service.start(FLUENTD_WINSVC_NAME)
+end
+
+exit 0 if early_exit
+
+require 'fluent/supervisor'
+if opts[:supervise]
+  Fluent::Supervisor.new(opts).run_supervisor
+else
+  if opts[:standalone_worker] && opts[:workers] && opts[:workers] > 1
+    puts "Error: multi workers is not supported with --no-supervisor"
+    exit 2
+  end
+  Fluent::Supervisor.new(opts).run_worker
+end

data/lib/fluent/command/plugin_config_formatter.rb

@@ -0,0 +1,258 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require "erb"
+require "optparse"
+require "pathname"
+require "fluent/plugin"
+require "fluent/env"
+require "fluent/engine"
+require "fluent/system_config"
+require "fluent/config/element"
+
+class FluentPluginConfigFormatter
+
+  AVAILABLE_FORMATS = [:markdown, :txt, :json]
+  SUPPORTED_TYPES = [
+    "input", "output", "filter",
+    "buffer", "parser", "formatter", "storage"
+  ]
+
+  def initialize(argv = ARGV)
+    @argv = argv
+
+    @compact = false
+    @format = :markdown
+    @verbose = false
+    @libs = []
+    @plugin_dirs = []
+    @options = {}
+
+    prepare_option_parser
+  end
+
+  def call
+    parse_options!
+    init_libraries
+    @plugin = Fluent::Plugin.__send__("new_#{@plugin_type}", @plugin_name)
+    dumped_config = {}
+    if @plugin.class.respond_to?(:plugin_helpers)
+      dumped_config[:plugin_helpers] = @plugin.class.plugin_helpers
+    end
+    @plugin.class.ancestors.reverse_each do |plugin_class|
+      next unless plugin_class.respond_to?(:dump_config_definition)
+      unless @verbose
+        next if plugin_class.name =~ /::PluginHelper::/
+      end
+      dumped_config_definition = plugin_class.dump_config_definition
+      dumped_config[plugin_class.name] = dumped_config_definition unless dumped_config_definition.empty?
+    end
+    case @format
+    when :txt
+      puts dump_txt(dumped_config)
+    when :markdown
+      puts dump_markdown(dumped_config)
+    when :json
+      puts dump_json(dumped_config)
+    end
+  end
+
+  private
+
+  def dump_txt(dumped_config)
+    dumped = ""
+    plugin_helpers = dumped_config.delete(:plugin_helpers)
+    if plugin_helpers && !plugin_helpers.empty?
+      dumped << "helpers: #{plugin_helpers.join(',')}\n"
+    end
+    if @verbose
+      dumped_config.each do |name, config|
+        dumped << "#{name}\n"
+        dumped << dump_section_txt(config)
+      end
+    else
+      configs = dumped_config.values
+      root_section = configs.shift
+      configs.each do |config|
+        root_section.update(config)
+      end
+      dumped << dump_section_txt(root_section)
+    end
+    dumped
+  end
+
+  def dump_section_txt(base_section, level = 0)
+    dumped = ""
+    indent = " " * level
+    if base_section[:section]
+      sections = []
+      params = base_section
+    else
+      sections, params = base_section.partition {|_name, value| value[:section] }
+    end
+    params.each do |name, config|
+      next if name == :section
+      dumped << "#{indent}#{name}: #{config[:type]}: (#{config[:default].inspect})"
+      dumped << " # #{config[:description]}" if config.key?(:description)
+      dumped << "\n"
+    end
+    sections.each do |section_name, sub_section|
+      required = sub_section.delete(:required)
+      multi = sub_section.delete(:multi)
+      alias_name = sub_section.delete(:alias)
+      required_label = required ? "required" : "optional"
+      multi_label = multi ? "multiple" : "single"
+      alias_label = "alias: #{alias_name}"
+      dumped << "#{indent}<#{section_name}>: #{required_label}, #{multi_label}"
+      if alias_name
+        dumped << ", #{alias_label}\n"
+      else
+        dumped << "\n"
+      end
+      sub_section.delete(:section)
+      dumped << dump_section_txt(sub_section, level + 1)
+    end
+    dumped
+  end
+
+  def dump_markdown(dumped_config)
+    dumped = ""
+    plugin_helpers = dumped_config.delete(:plugin_helpers)
+    if plugin_helpers && !plugin_helpers.empty?
+      dumped = "## Plugin helpers\n\n"
+      plugin_helpers.each do |plugin_helper|
+        dumped << "* #{plugin_helper}\n"
+      end
+      dumped << "\n"
+    end
+    dumped_config.each do |name, config|
+      if name == @plugin.class.name
+        dumped << "## #{name}\n\n"
+        dumped << dump_section_markdown(config)
+      else
+        dumped << "* See also: #{name}\n\n"
+      end
+    end
+    dumped
+  end
+
+  def dump_section_markdown(base_section, level = 0)
+    dumped = ""
+    if base_section[:section]
+      sections = []
+      params = base_section
+    else
+      sections, params = base_section.partition {|_name, value| value[:section] }
+    end
+    params.each do |name, config|
+      next if name == :section
+      template_name = @compact ? "param.md-compact.erb" : "param.md.erb"
+      dumped << ERB.new(template_path(template_name).read, nil, "-").result(binding)
+    end
+    dumped << "\n"
+    sections.each do |section_name, sub_section|
+      required = sub_section.delete(:required)
+      multi = sub_section.delete(:multi)
+      alias_name = sub_section.delete(:alias)
+      $log.trace(name: section_name, required: required, multi: multi, alias_name: alias_name)
+      sub_section.delete(:section)
+      dumped << ERB.new(template_path("section.md.erb").read, nil, "-").result(binding)
+    end
+    dumped
+  end
+
+  def dump_json(dumped_config)
+    if @compact
+      JSON.generate(dumped_config)
+    else
+      JSON.pretty_generate(dumped_config)
+    end
+  end
+
+  def usage(message = nil)
+    puts @parser.to_s
+    puts
+    puts "Error: #{message}" if message
+    exit(false)
+  end
+
+  def prepare_option_parser
+    @parser = OptionParser.new
+    @parser.banner = <<BANNER
+Usage: #{$0} [options] <type> <name>
+
+Output plugin config definitions
+
+Arguments:
+\ttype: #{SUPPORTED_TYPES.join(",")}
+\tname: registered plugin name
+
+Options:
+BANNER
+    @parser.on("--verbose", "Be verbose") do
+      @verbose = true
+    end
+    @parser.on("-c", "--compact", "Compact output") do
+      @compact = true
+    end
+    @parser.on("-f", "--format=FORMAT", "Specify format. (#{AVAILABLE_FORMATS.join(',')})") do |s|
+      format = s.to_sym
+      usage("Unsupported format: #{s}") unless AVAILABLE_FORMATS.include?(format)
+      @format = format
+    end
+    @parser.on("-I PATH", "Add PATH to $LOAD_PATH") do |s|
+      $LOAD_PATH.unshift(s)
+    end
+    @parser.on("-r NAME", "Load library") do |s|
+      @libs << s
+    end
+    @parser.on("-p", "--plugin=DIR", "Add plugin directory") do |s|
+      @plugin_dirs << s
+    end
+  end
+
+  def parse_options!
+    @parser.parse!(@argv)
+
+    raise "Must specify plugin type and name" unless @argv.size == 2
+
+    @plugin_type, @plugin_name = @argv
+    @options = {
+      compact: @compact,
+      format: @format,
+      verbose: @verbose,
+    }
+  rescue => e
+    usage(e)
+  end
+
+  def init_libraries
+    @libs.each do |lib|
+      require lib
+    end
+
+    @plugin_dirs.each do |dir|
+      if Dir.exist?(dir)
+        dir = File.expand_path(dir)
+        Fluent::Plugin.add_plugin_dir(dir)
+      end
+    end
+  end
+
+  def template_path(name)
+    (Pathname(__dir__) + "../../../templates/plugin_config_formatter/#{name}").realpath
+  end
+end