fluentd 0.12.40 → 1.6.2

data/lib/fluent/command/binlog_reader.rb

@@ -0,0 +1,244 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'optparse'
+require 'msgpack'
+
+require 'fluent/msgpack_factory'
+require 'fluent/formatter'
+require 'fluent/plugin'
+require 'fluent/config/element'
+require 'fluent/engine'
+require 'fluent/version'
+
+class FluentBinlogReader
+  SUBCOMMAND = %w(cat head formats)
+  HELP_TEXT = <<HELP
+Usage: fluent-binlog-reader <command> [<args>]
+
+Commands of fluent-binlog-reader:
+   cat     :     Read files sequentially, writing them to standard output.
+   head    :     Display the beginning of a text file.
+   formats :     Display plugins that you can use.
+
+See 'fluent-binlog-reader <command> --help' for more information on a specific command.
+HELP
+
+  def initialize(argv = ARGV)
+    @argv = argv
+  end
+
+  def call
+    command_class = BinlogReaderCommand.const_get(command)
+    command_class.new(@argv).call
+  end
+
+  private
+
+  def command
+    command = @argv.shift
+    if command
+      if command == '--version'
+        puts "#{File.basename($PROGRAM_NAME)} #{Fluent::VERSION}"
+        exit 0
+      elsif !SUBCOMMAND.include?(command)
+        usage "'#{command}' is not supported: Required subcommand : #{SUBCOMMAND.join(' | ')}"
+      end
+    else
+      usage "Required subcommand : #{SUBCOMMAND.join(' | ')}"
+    end
+
+    command.split('_').map(&:capitalize).join('')
+  end
+
+  def usage(msg = nil)
+    puts HELP_TEXT
+    puts "Error: #{msg}" if msg
+    exit 1
+  end
+end
+
+module BinlogReaderCommand
+  class Base
+    def initialize(argv = ARGV)
+      @argv = argv
+
+      @options = { plugin: [] }
+      @opt_parser = OptionParser.new do |opt|
+        opt.version = Fluent::VERSION
+        opt.separator 'Options:'
+
+        opt.on('-p DIR', '--plugin', 'add library directory path') do |v|
+          @options[:plugin] << v
+        end
+      end
+    end
+
+    def call
+      raise NotImplementedError, 'BUG: command  MUST implement this method'
+    end
+
+    private
+
+    def usage(msg = nil)
+      puts @opt_parser.to_s
+      puts "Error: #{msg}" if msg
+      exit 1
+    end
+
+    def parse_options!
+      @opt_parser.parse!(@argv)
+
+      unless @options[:plugin].empty?
+        if dir = @options[:plugin].find { |d| !Dir.exist?(d) }
+          usage "Directory #{dir} doesn't exist"
+        else
+          @options[:plugin].each do |d|
+            Fluent::Plugin.add_plugin_dir(d)
+          end
+        end
+      end
+    rescue => e
+      usage e
+    end
+  end
+
+  module Formattable
+    DEFAULT_OPTIONS = {
+      format: :out_file
+    }
+
+    def initialize(argv = ARGV)
+      super
+      @options.merge!(DEFAULT_OPTIONS)
+      configure_option_parser
+    end
+
+    private
+
+    def configure_option_parser
+      @options.merge!(config_params: {})
+
+      @opt_parser.banner = "Usage: fluent-binlog-reader #{self.class.to_s.split('::').last.downcase} [options] file"
+
+      @opt_parser.on('-f TYPE', '--format', 'configure output format') do |v|
+        @options[:format] = v.to_sym
+      end
+
+      @opt_parser.on('-e KEY=VALUE', 'configure formatter config params') do |v|
+        key, value = v.split('=')
+        usage "#{v} is invalid. valid format is like `key=value`" unless value
+        @options[:config_params].merge!(key => value)
+      end
+    end
+
+    def lookup_formatter(format, params)
+      conf = Fluent::Config::Element.new('ROOT', '', params, [])
+      formatter = Fluent::Plugin.new_formatter(format)
+
+      if formatter.respond_to?(:configure)
+        formatter.configure(conf)
+      end
+      formatter
+    rescue => e
+      usage e
+    end
+  end
+
+  class Head < Base
+    include Formattable
+
+    DEFAULT_HEAD_OPTIONS = {
+      count: 5
+    }
+
+    def initialize(argv = ARGV)
+      super
+      @options.merge!(default_options)
+      parse_options!
+    end
+
+    def call
+      @formatter = lookup_formatter(@options[:format], @options[:config_params])
+
+      File.open(@path, 'rb') do |io|
+        i = 1
+        Fluent::MessagePackFactory.unpacker(io).each do |(time, record)|
+          print @formatter.format(@path, time, record) # path is used for tag
+          break if @options[:count] && i == @options[:count]
+          i += 1
+        end
+      end
+    end
+
+    private
+
+    def default_options
+      DEFAULT_HEAD_OPTIONS
+    end
+
+    def parse_options!
+      @opt_parser.on('-n COUNT', 'Set the number of lines to display') do |v|
+        @options[:count] = v.to_i
+        usage "illegal line count -- #{@options[:count]}" if @options[:count] < 1
+      end
+
+      super
+
+      usage 'Path is required' if @argv.empty?
+      @path = @argv.first
+      usage "#{@path} is not found" unless File.exist?(@path)
+    end
+  end
+
+  class Cat < Head
+    DEFAULT_CAT_OPTIONS = {
+      count: nil                # Overwrite DEFAULT_HEAD_OPTIONS[:count]
+    }
+
+    def default_options
+      DEFAULT_CAT_OPTIONS
+    end
+  end
+
+  class Formats < Base
+    def initialize(argv = ARGV)
+      super
+      parse_options!
+    end
+
+    def call
+      prefix = Fluent::Plugin::FORMATTER_REGISTRY.dir_search_prefix || 'formatter_'
+
+      plugin_dirs = @options[:plugin]
+      unless plugin_dirs.empty?
+        plugin_dirs.each do |d|
+          Dir.glob("#{d}/#{prefix}*.rb").each do |path|
+            require File.absolute_path(path)
+          end
+        end
+      end
+
+      $LOAD_PATH.map do |lp|
+        Dir.glob("#{lp}/#{prefix}*.rb").each do |path|
+          require path
+        end
+      end
+
+      puts Fluent::Plugin::FORMATTER_REGISTRY.map.keys
+    end
+  end
+end

data/lib/fluent/command/ca_generate.rb

@@ -0,0 +1,181 @@
+require 'openssl'
+require 'optparse'
+require 'fileutils'
+require 'fluent/version'
+
+module Fluent
+  class CaGenerate
+    DEFAULT_OPTIONS = {
+      private_key_length: 2048,
+      cert_country:  'US',
+      cert_state:    'CA',
+      cert_locality: 'Mountain View',
+      cert_common_name: 'Fluentd Forward CA',
+    }
+    HELP_TEXT = <<HELP
+Usage: fluent-ca-generate DIR_PATH PRIVATE_KEY_PASSPHRASE [--country COUNTRY] [--state STATE] [--locality LOCALITY] [--common-name COMMON_NAME]
+HELP
+
+    def initialize(argv = ARGV)
+      @argv = argv
+      @options = {}
+      @opt_parser = OptionParser.new
+      configure_option_parser
+      @options.merge!(DEFAULT_OPTIONS)
+      parse_options!
+    end
+
+    def usage(msg = nil)
+      puts HELP_TEXT
+      puts "Error: #{msg}" if msg
+      exit 1
+    end
+
+    def call
+      ca_dir, passphrase, = @argv[0..1]
+
+      unless ca_dir && passphrase
+        puts "#{HELP_TEXT}"
+        puts ''
+        exit 1
+      end
+
+      FileUtils.mkdir_p(ca_dir)
+
+      cert, key = Fluent::CaGenerate.generate_ca_pair(@options)
+
+      key_data = key.export(OpenSSL::Cipher.new('aes256'), passphrase)
+      File.open(File.join(ca_dir, 'ca_key.pem'), 'w') do |file|
+        file.write key_data
+      end
+      File.open(File.join(ca_dir, 'ca_cert.pem'), 'w') do |file|
+        file.write cert.to_pem
+      end
+
+      puts "successfully generated: ca_key.pem, ca_cert.pem"
+      puts "copy and use ca_cert.pem to client(out_forward)"
+    end
+
+    def self.certificates_from_file(path)
+      data = File.read(path)
+      pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\n(?:[^-]*\n)+-+END CERTIFICATE-+\n', Regexp::MULTILINE)
+      list = []
+      data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match)}
+      list
+    end
+
+    def self.generate_ca_pair(opts={})
+      key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+      issuer = subject = OpenSSL::X509::Name.new
+      subject.add_entry('C', opts[:cert_country])
+      subject.add_entry('ST', opts[:cert_state])
+      subject.add_entry('L', opts[:cert_locality])
+      subject.add_entry('CN', opts[:cert_common_name])
+
+      digest = OpenSSL::Digest::SHA256.new
+
+      cert = OpenSSL::X509::Certificate.new
+      cert.not_before = Time.at(0)
+      cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after
+      cert.public_key = key
+      cert.serial = 1
+      cert.issuer = issuer
+      cert.subject = subject
+      cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(true)]))
+      cert.sign(key, digest)
+
+      return cert, key
+    end
+
+    def self.generate_server_pair(opts={})
+      key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+      ca_key = OpenSSL::PKey::read(File.read(opts[:ca_key_path]), opts[:ca_key_passphrase])
+      ca_cert = OpenSSL::X509::Certificate.new(File.read(opts[:ca_cert_path]))
+      issuer = ca_cert.issuer
+
+      subject = OpenSSL::X509::Name.new
+      subject.add_entry('C', opts[:country])
+      subject.add_entry('ST', opts[:state])
+      subject.add_entry('L', opts[:locality])
+      subject.add_entry('CN', opts[:common_name])
+
+      digest = OpenSSL::Digest::SHA256.new
+
+      cert = OpenSSL::X509::Certificate.new
+      cert.not_before = Time.at(0)
+      cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after
+      cert.public_key = key
+      cert.serial = 2
+      cert.issuer = issuer
+      cert.subject = subject
+
+      cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
+      cert.add_extension OpenSSL::X509::Extension.new('nsCertType', 'server')
+
+      cert.sign ca_key, digest
+
+      return cert, key
+    end
+
+    def self.generate_self_signed_server_pair(opts={})
+      key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+      issuer = subject = OpenSSL::X509::Name.new
+      subject.add_entry('C', opts[:country])
+      subject.add_entry('ST', opts[:state])
+      subject.add_entry('L', opts[:locality])
+      subject.add_entry('CN', opts[:common_name])
+
+      digest = OpenSSL::Digest::SHA256.new
+
+      cert = OpenSSL::X509::Certificate.new
+      cert.not_before = Time.at(0)
+      cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after
+      cert.public_key = key
+      cert.serial = 1
+      cert.issuer = issuer
+      cert.subject  = subject
+      cert.sign(key, digest)
+
+      return cert, key
+    end
+
+    private
+
+    def configure_option_parser
+      @opt_parser.banner = HELP_TEXT
+      @opt_parser.version = Fluent::VERSION
+
+      @opt_parser.on('--key-length [KEY_LENGTH]',
+                     "configure key length. (default: #{DEFAULT_OPTIONS[:private_key_length]})") do |v|
+        @options[:private_key_length] = v.to_i
+      end
+
+      @opt_parser.on('--country [COUNTRY]',
+                     "configure country. (default: #{DEFAULT_OPTIONS[:cert_country]})") do |v|
+        @options[:cert_country] = v.upcase
+      end
+
+      @opt_parser.on('--state [STATE]',
+                     "configure state. (default: #{DEFAULT_OPTIONS[:cert_state]})") do |v|
+        @options[:cert_state] = v
+      end
+
+      @opt_parser.on('--locality [LOCALITY]',
+                     "configure locality. (default: #{DEFAULT_OPTIONS[:cert_locality]})") do |v|
+        @options[:cert_locality] = v
+      end
+
+      @opt_parser.on('--common-name [COMMON_NAME]',
+                     "configure common name (default: #{DEFAULT_OPTIONS[:cert_common_name]})") do |v|
+        @options[:cert_common_name] = v
+      end
+    end
+
+    def parse_options!
+      @opt_parser.parse!(@argv)
+    end
+  end
+end

data/lib/fluent/command/cat.rb

@@ -16,12 +16,16 @@
 
 require 'optparse'
 require 'fluent/env'
+require 'fluent/time'
+require 'fluent/msgpack_factory'
+require 'fluent/version'
 
 op = OptionParser.new
 
 op.banner += " <tag>"
+op.version = Fluent::VERSION
 
-port = Fluent::DEFAULT_LISTEN_PORT
+port = 24224
 host = '127.0.0.1'
 unix = false
 socket_path = Fluent::DEFAULT_SOCKET_PATH
@@ -29,6 +33,8 @@ socket_path = Fluent::DEFAULT_SOCKET_PATH
 config_path = Fluent::DEFAULT_CONFIG_PATH
 format = 'json'
 message_key = 'message'
+time_as_integer = false
+retry_limit = 5
 
 op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
   port = i
@@ -66,7 +72,15 @@ op.on('--message-key KEY', "key field for none format (default: #{message_key})"
   message_key = s
 }
 
-(class << self; self; end).module_eval do
+op.on('--time-as-integer', "Send time as integer for v0.12 or earlier", TrueClass) { |b|
+  time_as_integer = true
+}
+
+op.on('--retry-limit N', "Specify the number of retry limit (default: #{retry_limit})", Integer) {|n|
+  retry_limit = n
+}
+
+singleton_class.module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
     puts "error: #{msg}" if msg
@@ -98,6 +112,8 @@ require 'msgpack'
 class Writer
   include MonitorMixin
 
+  RetryLimitError = Class.new(StandardError)
+
   class TimerThread
     def initialize(writer)
       @writer = writer
@@ -121,7 +137,7 @@ class Writer
     end
   end
 
-  def initialize(tag, connector)
+  def initialize(tag, connector, time_as_integer: false, retry_limit: 5)
     @tag = tag
     @connector = connector
     @socket = false
@@ -133,7 +149,8 @@ class Writer
     @pending = []
     @pending_limit = 1024  # TODO
     @retry_wait = 1
-    @retry_limit = 5  # TODO
+    @retry_limit = retry_limit
+    @time_as_integer = time_as_integer
 
     super()
   end
@@ -143,7 +160,9 @@ class Writer
       raise ArgumentError, "Input must be a map (got #{record.class})"
     end
 
-    entry = [Time.now.to_i, record]
+    time = Fluent::EventTime.now
+    time = time.to_i if @time_as_integer
+    entry = [time, record]
     synchronize {
       unless write_impl([entry])
         # write failed
@@ -200,7 +219,8 @@ class Writer
     end
 
     begin
-      socket.write [@tag, array].to_msgpack
+      packer = Fluent::MessagePackFactory.packer
+      socket.write packer.pack([@tag, array])
       socket.flush
     rescue
       $stderr.puts "write failed: #{$!}"
@@ -223,21 +243,24 @@ class Writer
   end
 
   def try_connect
-    now = Time.now.to_i
-
-    unless @error_history.empty?
-      # wait before re-connecting
-      wait = @retry_wait * (2 ** (@error_history.size-1))
-      if now <= @socket_time + wait
-        return false
+    begin
+      now = Time.now.to_i
+
+      unless @error_history.empty?
+        # wait before re-connecting
+        wait = 1 #@retry_wait * (2 ** (@error_history.size-1))
+        if now <= @socket_time + wait
+          sleep(wait)
+          try_connect
+        end
       end
-    end
 
-    begin
       @socket = @connector.call
       @error_history.clear
       return true
 
+    rescue RetryLimitError => ex
+      raise ex
     rescue
       $stderr.puts "connect failed: #{$!}"
       @error_history << $!
@@ -250,9 +273,10 @@ class Writer
         }
         @pending.clear
         @error_history.clear
+        raise RetryLimitError, "exceed retry limit"
+      else
+        retry
       end
-
-      return false
     end
   end
 
@@ -272,7 +296,7 @@ else
   }
 end
 
-w = Writer.new(tag, connector)
+w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit)
 w.start
 
 case format

data/lib/fluent/command/debug.rb

@@ -34,7 +34,7 @@ op.on('-u', '--unix PATH', "use unix socket instead of tcp") {|b|
   unix = b
 }
 
-(class << self; self; end).module_eval do
+singleton_class.module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
     puts "error: #{msg}" if msg
@@ -62,13 +62,16 @@ end
 
 require 'fluent/log'
 require 'fluent/engine'
-require 'fluent/config'
 require 'fluent/system_config'
+require 'serverengine'
 
-$log = Fluent::Log.new(STDERR, Fluent::Log::LEVEL_TRACE)
-conf = Fluent::Config.parse('', 'fluent-debug.conf', '', true)
-system_conf = Fluent::SystemConfig.create(conf)
-Fluent::Engine.init(system_conf)
+include Fluent::SystemConfig::Mixin
+
+dl_opts = {}
+dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE
+logger = ServerEngine::DaemonLogger.new(STDERR, dl_opts)
+$log = Fluent::Log.new(logger)
+Fluent::Engine.init(system_config)
 
 DRb::DRbObject.class_eval do
   undef_method :methods
@@ -88,13 +91,12 @@ include Fluent
 
 puts "Connected to #{uri}."
 puts "Usage:"
-puts "    Engine.match('some.tag').output  : get an output plugin instance"
-puts "    Engine.sources[i]                : get input plugin instances"
-puts "    Plugin.load_plugin(type,name)    : load plugin class (use this if you get DRb::DRbUnknown)"
+puts "    Fluent::Engine.root_agent.event_router.match('some.tag') : get an output plugin instance"
+puts "    Fluent::Engine.root_agent.inputs[i]                      : get input plugin instances"
+puts "    Fluent::Plugin::OUTPUT_REGISTRY.lookup(name)             : load output plugin class (use this if you get DRb::DRbUnknown)"
 puts ""
 
 Encoding.default_internal = nil if Encoding.respond_to?(:default_internal)
 
 require 'irb'
 IRB.start
-