fluentd 0.14.17-x64-mingw32 → 1.3.1-x64-mingw32

data/lib/fluent/command/ca_generate.rb

@@ -0,0 +1,181 @@
+require 'openssl'
+require 'optparse'
+require 'fileutils'
+require 'fluent/version'
+
+module Fluent
+  class CaGenerate
+    DEFAULT_OPTIONS = {
+      private_key_length: 2048,
+      cert_country:  'US',
+      cert_state:    'CA',
+      cert_locality: 'Mountain View',
+      cert_common_name: 'Fluentd Forward CA',
+    }
+    HELP_TEXT = <<HELP
+Usage: fluent-ca-generate DIR_PATH PRIVATE_KEY_PASSPHRASE [--country COUNTRY] [--state STATE] [--locality LOCALITY] [--common-name COMMON_NAME]
+HELP
+
+    def initialize(argv = ARGV)
+      @argv = argv
+      @options = {}
+      @opt_parser = OptionParser.new
+      configure_option_parser
+      @options.merge!(DEFAULT_OPTIONS)
+      parse_options!
+    end
+
+    def usage(msg = nil)
+      puts HELP_TEXT
+      puts "Error: #{msg}" if msg
+      exit 1
+    end
+
+    def call
+      ca_dir, passphrase, = @argv[0..1]
+
+      unless ca_dir && passphrase
+        puts "#{HELP_TEXT}"
+        puts ''
+        exit 1
+      end
+
+      FileUtils.mkdir_p(ca_dir)
+
+      cert, key = Fluent::CaGenerate.generate_ca_pair(@options)
+
+      key_data = key.export(OpenSSL::Cipher.new('aes256'), passphrase)
+      File.open(File.join(ca_dir, 'ca_key.pem'), 'w') do |file|
+        file.write key_data
+      end
+      File.open(File.join(ca_dir, 'ca_cert.pem'), 'w') do |file|
+        file.write cert.to_pem
+      end
+
+      puts "successfully generated: ca_key.pem, ca_cert.pem"
+      puts "copy and use ca_cert.pem to client(out_forward)"
+    end
+
+    def self.certificates_from_file(path)
+      data = File.read(path)
+      pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\n(?:[^-]*\n)+-+END CERTIFICATE-+\n', Regexp::MULTILINE)
+      list = []
+      data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match)}
+      list
+    end
+
+    def self.generate_ca_pair(opts={})
+      key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+      issuer = subject = OpenSSL::X509::Name.new
+      subject.add_entry('C', opts[:cert_country])
+      subject.add_entry('ST', opts[:cert_state])
+      subject.add_entry('L', opts[:cert_locality])
+      subject.add_entry('CN', opts[:cert_common_name])
+
+      digest = OpenSSL::Digest::SHA256.new
+
+      cert = OpenSSL::X509::Certificate.new
+      cert.not_before = Time.at(0)
+      cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after
+      cert.public_key = key
+      cert.serial = 1
+      cert.issuer = issuer
+      cert.subject = subject
+      cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(true)]))
+      cert.sign(key, digest)
+
+      return cert, key
+    end
+
+    def self.generate_server_pair(opts={})
+      key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+      ca_key = OpenSSL::PKey::RSA.new(File.read(opts[:ca_key_path]), opts[:ca_key_passphrase])
+      ca_cert = OpenSSL::X509::Certificate.new(File.read(opts[:ca_cert_path]))
+      issuer = ca_cert.issuer
+
+      subject = OpenSSL::X509::Name.new
+      subject.add_entry('C', opts[:country])
+      subject.add_entry('ST', opts[:state])
+      subject.add_entry('L', opts[:locality])
+      subject.add_entry('CN', opts[:common_name])
+
+      digest = OpenSSL::Digest::SHA256.new
+
+      cert = OpenSSL::X509::Certificate.new
+      cert.not_before = Time.at(0)
+      cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after
+      cert.public_key = key
+      cert.serial = 2
+      cert.issuer = issuer
+      cert.subject = subject
+
+      cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
+      cert.add_extension OpenSSL::X509::Extension.new('nsCertType', 'server')
+
+      cert.sign ca_key, digest
+
+      return cert, key
+    end
+
+    def self.generate_self_signed_server_pair(opts={})
+      key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+      issuer = subject = OpenSSL::X509::Name.new
+      subject.add_entry('C', opts[:country])
+      subject.add_entry('ST', opts[:state])
+      subject.add_entry('L', opts[:locality])
+      subject.add_entry('CN', opts[:common_name])
+
+      digest = OpenSSL::Digest::SHA256.new
+
+      cert = OpenSSL::X509::Certificate.new
+      cert.not_before = Time.at(0)
+      cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after
+      cert.public_key = key
+      cert.serial = 1
+      cert.issuer = issuer
+      cert.subject  = subject
+      cert.sign(key, digest)
+
+      return cert, key
+    end
+
+    private
+
+    def configure_option_parser
+      @opt_parser.banner = HELP_TEXT
+      @opt_parser.version = Fluent::VERSION
+
+      @opt_parser.on('--key-length [KEY_LENGTH]',
+                     "configure key length. (default: #{DEFAULT_OPTIONS[:private_key_length]})") do |v|
+        @options[:private_key_length] = v.to_i
+      end
+
+      @opt_parser.on('--country [COUNTRY]',
+                     "configure country. (default: #{DEFAULT_OPTIONS[:cert_country]})") do |v|
+        @options[:cert_country] = v.upcase
+      end
+
+      @opt_parser.on('--state [STATE]',
+                     "configure state. (default: #{DEFAULT_OPTIONS[:cert_state]})") do |v|
+        @options[:cert_state] = v
+      end
+
+      @opt_parser.on('--locality [LOCALITY]',
+                     "configure locality. (default: #{DEFAULT_OPTIONS[:cert_locality]})") do |v|
+        @options[:cert_locality] = v
+      end
+
+      @opt_parser.on('--common-name [COMMON_NAME]',
+                     "configure common name (default: #{DEFAULT_OPTIONS[:cert_common_name]})") do |v|
+        @options[:cert_common_name] = v
+      end
+    end
+
+    def parse_options!
+      @opt_parser.parse!(@argv)
+    end
+  end
+end

data/lib/fluent/command/cat.rb

@@ -18,10 +18,12 @@ require 'optparse'
 require 'fluent/env'
 require 'fluent/time'
 require 'fluent/msgpack_factory'
+require 'fluent/version'
 
 op = OptionParser.new
 
 op.banner += " <tag>"
+op.version = Fluent::VERSION
 
 port = 24224
 host = '127.0.0.1'
@@ -32,6 +34,7 @@ config_path = Fluent::DEFAULT_CONFIG_PATH
 format = 'json'
 message_key = 'message'
 time_as_integer = false
+retry_limit = 5
 
 op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
   port = i
@@ -73,7 +76,11 @@ op.on('--time-as-integer', "Send time as integer for v0.12 or earlier", TrueClas
   time_as_integer = true
 }
 
-(class << self; self; end).module_eval do
+op.on('--retry-limit N', "Specify the number of retry limit (default: #{retry_limit})", Integer) {|n|
+  retry_limit = n
+}
+
+singleton_class.module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
     puts "error: #{msg}" if msg
@@ -105,6 +112,8 @@ require 'msgpack'
 class Writer
   include MonitorMixin
 
+  RetryLimitError = Class.new(StandardError)
+
   class TimerThread
     def initialize(writer)
       @writer = writer
@@ -128,7 +137,7 @@ class Writer
     end
   end
 
-  def initialize(tag, connector, time_as_integer: false)
+  def initialize(tag, connector, time_as_integer: false, retry_limit: 5)
     @tag = tag
     @connector = connector
     @socket = false
@@ -140,7 +149,7 @@ class Writer
     @pending = []
     @pending_limit = 1024  # TODO
     @retry_wait = 1
-    @retry_limit = 5  # TODO
+    @retry_limit = retry_limit
     @time_as_integer = time_as_integer
 
     super()
@@ -234,21 +243,24 @@ class Writer
   end
 
   def try_connect
-    now = Time.now.to_i
-
-    unless @error_history.empty?
-      # wait before re-connecting
-      wait = @retry_wait * (2 ** (@error_history.size-1))
-      if now <= @socket_time + wait
-        return false
+    begin
+      now = Time.now.to_i
+
+      unless @error_history.empty?
+        # wait before re-connecting
+        wait = 1 #@retry_wait * (2 ** (@error_history.size-1))
+        if now <= @socket_time + wait
+          sleep(wait)
+          try_connect
+        end
       end
-    end
 
-    begin
       @socket = @connector.call
       @error_history.clear
       return true
 
+    rescue RetryLimitError => ex
+      raise ex
     rescue
       $stderr.puts "connect failed: #{$!}"
       @error_history << $!
@@ -261,9 +273,10 @@ class Writer
         }
         @pending.clear
         @error_history.clear
+        raise RetryLimitError, "exceed retry limit"
+      else
+        retry
       end
-
-      return false
     end
   end
 
@@ -283,7 +296,7 @@ else
   }
 end
 
-w = Writer.new(tag, connector, time_as_integer: time_as_integer)
+w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit)
 w.start
 
 case format

data/lib/fluent/command/debug.rb

@@ -34,7 +34,7 @@ op.on('-u', '--unix PATH', "use unix socket instead of tcp") {|b|
   unix = b
 }
 
-(class << self; self; end).module_eval do
+singleton_class.module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
     puts "error: #{msg}" if msg
@@ -91,9 +91,9 @@ include Fluent
 
 puts "Connected to #{uri}."
 puts "Usage:"
-puts "    Fluent::Engine.match('some.tag').output  : get an output plugin instance"
-puts "    Fluent::Engine.sources[i]                : get input plugin instances"
-puts "    Fluent::Plugin.load_plugin(type,name)    : load plugin class (use this if you get DRb::DRbUnknown)"
+puts "    Fluent::Engine.root_agent.event_router.match('some.tag') : get an output plugin instance"
+puts "    Fluent::Engine.root_agent.inputs[i]                      : get input plugin instances"
+puts "    Fluent::Plugin::OUTPUT_REGISTRY.lookup(name)             : load output plugin class (use this if you get DRb::DRbUnknown)"
 puts ""
 
 Encoding.default_internal = nil if Encoding.respond_to?(:default_internal)

data/lib/fluent/command/fluentd.rb

@@ -177,7 +177,7 @@ if Fluent.windows?
     opts[:regwinsvcautostart] = s
   }
 
-  op.on('--reg-winsvc-fluentdopt OPTION', "specify fluentd option paramters for Windows Service. (Windows only)") {|s|
+  op.on('--reg-winsvc-fluentdopt OPTION', "specify fluentd option parameters for Windows Service. (Windows only)") {|s|
     opts[:fluentdopt] = s
   }
   
@@ -195,7 +195,7 @@ if Fluent.windows?
 end
 
 
-(class << self; self; end).module_eval do
+singleton_class.module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
     puts "error: #{msg}" if msg

data/lib/fluent/command/plugin_config_formatter.rb

@@ -22,6 +22,7 @@ require "fluent/env"
 require "fluent/engine"
 require "fluent/system_config"
 require "fluent/config/element"
+require 'fluent/version'
 
 class FluentPluginConfigFormatter
 
@@ -31,6 +32,8 @@ class FluentPluginConfigFormatter
     "buffer", "parser", "formatter", "storage"
   ]
 
+  DOCS_BASE_URL = "https://docs.fluentd.org/v1.0/articles/"
+
   def initialize(argv = ARGV)
     @argv = argv
 
@@ -134,7 +137,7 @@ class FluentPluginConfigFormatter
     if plugin_helpers && !plugin_helpers.empty?
       dumped = "## Plugin helpers\n\n"
       plugin_helpers.each do |plugin_helper|
-        dumped << "* #{plugin_helper}\n"
+        dumped << "* #{plugin_helper_markdown_link(plugin_helper)}\n"
       end
       dumped << "\n"
     end
@@ -143,7 +146,7 @@ class FluentPluginConfigFormatter
         dumped << "## #{name}\n\n"
         dumped << dump_section_markdown(config)
       else
-        dumped << "* See also: #{name}\n\n"
+        dumped << "* See also: #{plugin_overview_markdown_link(name)}\n\n"
       end
     end
     dumped
@@ -181,6 +184,24 @@ class FluentPluginConfigFormatter
     end
   end
 
+  def plugin_helper_url(plugin_helper)
+    "#{DOCS_BASE_URL}api-plugin-helper-#{plugin_helper}"
+  end
+
+  def plugin_helper_markdown_link(plugin_helper)
+    "[#{plugin_helper}](#{plugin_helper_url(plugin_helper)})"
+  end
+
+  def plugin_overview_url(class_name)
+    plugin_type = class_name.slice(/::(\w+)\z/, 1).downcase
+    "#{DOCS_BASE_URL}#{plugin_type}-plugin-overview"
+  end
+
+  def plugin_overview_markdown_link(class_name)
+    plugin_type = class_name.slice(/::(\w+)\z/, 1)
+    "[#{plugin_type} Plugin Overview](#{plugin_overview_url(class_name)})"
+  end
+
   def usage(message = nil)
     puts @parser.to_s
     puts
@@ -190,6 +211,7 @@ class FluentPluginConfigFormatter
 
   def prepare_option_parser
     @parser = OptionParser.new
+    @parser.version = Fluent::VERSION
     @parser.banner = <<BANNER
 Usage: #{$0} [options] <type> <name>
 

data/lib/fluent/command/plugin_generator.rb

@@ -21,6 +21,7 @@ require "erb"
 require "open-uri"
 
 require "fluent/registry"
+require 'fluent/version'
 
 class FluentPluginGenerator
   attr_reader :type, :name
@@ -91,6 +92,7 @@ class FluentPluginGenerator
 
   def prepare_parser
     @parser = OptionParser.new
+    @parser.version = Fluent::VERSION
     @parser.banner = <<BANNER
 Usage: fluent-plugin-generate [options] <type> <name>
 
@@ -137,32 +139,36 @@ BANNER
   end
 
   def gem_name
-    "fluent-plugin-#{name}"
+    "fluent-plugin-#{dash_name}"
+  end
+
+  def plugin_name
+    underscore_name
   end
 
   def class_name
-    "#{name.capitalize}#{type.capitalize}"
+    "#{capitalized_name}#{type.capitalize}"
   end
 
   def plugin_filename
     case type
     when "input"
-      "in_#{name}.rb"
+      "in_#{underscore_name}.rb"
     when "output"
-      "out_#{name}.rb"
+      "out_#{underscore_name}.rb"
     else
-      "#{type}_#{name}.rb"
+      "#{type}_#{underscore_name}.rb"
     end
   end
 
   def test_filename
     case type
     when "input"
-      "test_in_#{name}.rb"
+      "test_in_#{underscore_name}.rb"
     when "output"
-      "test_out_#{name}.rb"
+      "test_out_#{underscore_name}.rb"
     else
-      "test_#{type}_#{name}.rb"
+      "test_#{type}_#{underscore_name}.rb"
     end
   end
 
@@ -179,6 +185,18 @@ BANNER
     end
   end
 
+  def capitalized_name
+    @capitalized_name ||= name.split(/[-_]/).map(&:capitalize).join
+  end
+
+  def underscore_name
+    @underscore_name ||= name.tr("-", "_")
+  end
+
+  def dash_name
+    @dash_name ||= name.tr("_", "-")
+  end
+
   def preamble
     @license.preamble(user_name)
   end