dtomasgu-fluentd 1.14.7.pre.dev

data/lib/fluent/command/ca_generate.rb

@@ -0,0 +1,184 @@
+require 'openssl'
+require 'optparse'
+require 'fileutils'
+require 'fluent/version'
+
+module Fluent
+  class CaGenerate
+    DEFAULT_OPTIONS = {
+      private_key_length: 2048,
+      cert_country:  'US',
+      cert_state:    'CA',
+      cert_locality: 'Mountain View',
+      cert_common_name: 'Fluentd Forward CA',
+    }
+    HELP_TEXT = <<HELP
+Usage: fluent-ca-generate DIR_PATH PRIVATE_KEY_PASSPHRASE [--country COUNTRY] [--state STATE] [--locality LOCALITY] [--common-name COMMON_NAME]
+HELP
+
+    def initialize(argv = ARGV)
+      @argv = argv
+      @options = {}
+      @opt_parser = OptionParser.new
+      configure_option_parser
+      @options.merge!(DEFAULT_OPTIONS)
+      parse_options!
+    end
+
+    def usage(msg = nil)
+      puts HELP_TEXT
+      puts "Error: #{msg}" if msg
+      exit 1
+    end
+
+    def call
+      ca_dir, passphrase, = @argv[0..1]
+
+      unless ca_dir && passphrase
+        puts "#{HELP_TEXT}"
+        puts ''
+        exit 1
+      end
+
+      FileUtils.mkdir_p(ca_dir)
+
+      cert, key = Fluent::CaGenerate.generate_ca_pair(@options)
+
+      key_data = key.export(OpenSSL::Cipher.new('aes256'), passphrase)
+      File.open(File.join(ca_dir, 'ca_key.pem'), 'w') do |file|
+        file.write key_data
+      end
+      File.open(File.join(ca_dir, 'ca_cert.pem'), 'w') do |file|
+        file.write cert.to_pem
+      end
+
+      puts "successfully generated: ca_key.pem, ca_cert.pem"
+      puts "copy and use ca_cert.pem to client(out_forward)"
+    end
+
+    def self.certificates_from_file(path)
+      data = File.read(path)
+      pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\n(?:[^-]*\n)+-+END CERTIFICATE-+\n', Regexp::MULTILINE)
+      list = []
+      data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match)}
+      list
+    end
+
+    def self.generate_ca_pair(opts={})
+      key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+      issuer = subject = OpenSSL::X509::Name.new
+      subject.add_entry('C', opts[:cert_country])
+      subject.add_entry('ST', opts[:cert_state])
+      subject.add_entry('L', opts[:cert_locality])
+      subject.add_entry('CN', opts[:cert_common_name])
+
+      digest = OpenSSL::Digest::SHA256.new
+
+      factory = OpenSSL::X509::ExtensionFactory.new
+
+      cert = OpenSSL::X509::Certificate.new
+      cert.not_before = Time.at(0)
+      cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after
+      cert.public_key = key
+      cert.serial = 1
+      cert.issuer = issuer
+      cert.subject = subject
+      cert.add_extension(factory.create_extension('basicConstraints', 'CA:TRUE'))
+      cert.sign(key, digest)
+
+      return cert, key
+    end
+
+    def self.generate_server_pair(opts={})
+      key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+      ca_key = OpenSSL::PKey::read(File.read(opts[:ca_key_path]), opts[:ca_key_passphrase])
+      ca_cert = OpenSSL::X509::Certificate.new(File.read(opts[:ca_cert_path]))
+      issuer = ca_cert.issuer
+
+      subject = OpenSSL::X509::Name.new
+      subject.add_entry('C', opts[:country])
+      subject.add_entry('ST', opts[:state])
+      subject.add_entry('L', opts[:locality])
+      subject.add_entry('CN', opts[:common_name])
+
+      digest = OpenSSL::Digest::SHA256.new
+
+      cert = OpenSSL::X509::Certificate.new
+      cert.not_before = Time.at(0)
+      cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after
+      cert.public_key = key
+      cert.serial = 2
+      cert.issuer = issuer
+      cert.subject = subject
+
+      factory = OpenSSL::X509::ExtensionFactory.new
+      server_cert.add_extension(factory.create_extension('basicConstraints', 'CA:FALSE'))
+      server_cert.add_extension(factory.create_extension('nsCertType', 'server'))
+
+      cert.sign ca_key, digest
+
+      return cert, key
+    end
+
+    def self.generate_self_signed_server_pair(opts={})
+      key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+      issuer = subject = OpenSSL::X509::Name.new
+      subject.add_entry('C', opts[:country])
+      subject.add_entry('ST', opts[:state])
+      subject.add_entry('L', opts[:locality])
+      subject.add_entry('CN', opts[:common_name])
+
+      digest = OpenSSL::Digest::SHA256.new
+
+      cert = OpenSSL::X509::Certificate.new
+      cert.not_before = Time.at(0)
+      cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after
+      cert.public_key = key
+      cert.serial = 1
+      cert.issuer = issuer
+      cert.subject  = subject
+      cert.sign(key, digest)
+
+      return cert, key
+    end
+
+    private
+
+    def configure_option_parser
+      @opt_parser.banner = HELP_TEXT
+      @opt_parser.version = Fluent::VERSION
+
+      @opt_parser.on('--key-length [KEY_LENGTH]',
+                     "configure key length. (default: #{DEFAULT_OPTIONS[:private_key_length]})") do |v|
+        @options[:private_key_length] = v.to_i
+      end
+
+      @opt_parser.on('--country [COUNTRY]',
+                     "configure country. (default: #{DEFAULT_OPTIONS[:cert_country]})") do |v|
+        @options[:cert_country] = v.upcase
+      end
+
+      @opt_parser.on('--state [STATE]',
+                     "configure state. (default: #{DEFAULT_OPTIONS[:cert_state]})") do |v|
+        @options[:cert_state] = v
+      end
+
+      @opt_parser.on('--locality [LOCALITY]',
+                     "configure locality. (default: #{DEFAULT_OPTIONS[:cert_locality]})") do |v|
+        @options[:cert_locality] = v
+      end
+
+      @opt_parser.on('--common-name [COMMON_NAME]',
+                     "configure common name (default: #{DEFAULT_OPTIONS[:cert_common_name]})") do |v|
+        @options[:cert_common_name] = v
+      end
+    end
+
+    def parse_options!
+      @opt_parser.parse!(@argv)
+    end
+  end
+end

data/lib/fluent/command/cap_ctl.rb

@@ -0,0 +1,174 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'optparse'
+require 'fluent/log'
+require 'fluent/env'
+require 'fluent/capability'
+
+module Fluent
+  class CapCtl
+    def prepare_option_parser
+      @op = OptionParser.new
+
+      @op.on('--clear', "Clear Fluentd Ruby capability") {|s|
+        @opts[:clear_capabilities] = true
+      }
+
+      @op.on('--add [CAPABILITITY1,CAPABILITY2, ...]', "Add capabilities into Fluentd Ruby") {|s|
+        @opts[:add_capabilities] = s
+      }
+
+      @op.on('--drop [CAPABILITITY1,CAPABILITY2, ...]', "Drop capabilities from Fluentd Ruby") {|s|
+        @opts[:drop_capabilities] = s
+      }
+
+      @op.on('--get', "Get capabilities for Fluentd Ruby") {|s|
+        @opts[:get_capabilities] = true
+      }
+
+      @op.on('-f', '--file FILE', "Specify target file to add Linux capabilities") {|s|
+        @opts[:target_file] = s
+      }
+    end
+
+    def usage(msg)
+      puts @op.to_s
+      puts "error: #{msg}" if msg
+      exit 1
+    end
+
+    def initialize(argv = ARGV)
+      @opts = {}
+      @argv = argv
+
+      if Fluent.linux?
+        begin
+          require 'capng'
+
+          @capng = CapNG.new
+        rescue LoadError
+          puts "Error: capng_c is not loaded. Please install it first."
+          exit 1
+        end
+      else
+        puts "Error: This environment is not supported."
+        exit 2
+      end
+
+      prepare_option_parser
+    end
+
+    def call
+      parse_options!(@argv)
+
+      target_file = if !!@opts[:target_file]
+                       @opts[:target_file]
+                     else
+                       File.readlink("/proc/self/exe")
+                     end
+
+      if @opts[:clear_capabilities]
+        clear_capabilities(@opts, target_file)
+      elsif @opts[:add_capabilities]
+        add_capabilities(@opts, target_file)
+      elsif @opts[:drop_capabilities]
+        drop_capabilities(@opts, target_file)
+      end
+      if @opts[:get_capabilities]
+        get_capabilities(@opts, target_file)
+      end
+    end
+
+    def clear_capabilities(opts, target_file)
+      if !!opts[:clear_capabilities]
+        @capng.clear(:caps)
+        ret = @capng.apply_caps_file(target_file)
+        puts "Clear capabilities #{ret ? 'done' : 'fail'}."
+      end
+    end
+
+    def add_capabilities(opts, target_file)
+      if add_caps = opts[:add_capabilities]
+        @capng.clear(:caps)
+        @capng.caps_file(target_file)
+        capabilities = add_caps.split(/\s*,\s*/)
+        check_capabilities(capabilities, get_valid_capabilities)
+        ret = @capng.update(:add,
+                            CapNG::Type::EFFECTIVE | CapNG::Type::INHERITABLE | CapNG::Type::PERMITTED,
+                            capabilities)
+        puts "Updating #{add_caps} #{ret ? 'done' : 'fail'}."
+        ret = @capng.apply_caps_file(target_file)
+        puts "Adding #{add_caps} #{ret ? 'done' : 'fail'}."
+      end
+    end
+
+    def drop_capabilities(opts, target_file)
+      if drop_caps = opts[:drop_capabilities]
+        @capng.clear(:caps)
+        @capng.caps_file(target_file)
+        capabilities = drop_caps.split(/\s*,\s*/)
+        check_capabilities(capabilities, get_valid_capabilities)
+        ret = @capng.update(:drop,
+                            CapNG::Type::EFFECTIVE | CapNG::Type::INHERITABLE | CapNG::Type::PERMITTED,
+                            capabilities)
+        puts "Updating #{drop_caps} #{ret ? 'done' : 'fail'}."
+        @capng.apply_caps_file(target_file)
+        puts "Dropping #{drop_caps} #{ret ? 'done' : 'fail'}."
+      end
+    end
+
+    def get_capabilities(opts, target_file)
+      if opts[:get_capabilities]
+        @capng.caps_file(target_file)
+        print = CapNG::Print.new
+        puts "Capabilities in '#{target_file}',"
+        puts "Effective:   #{print.caps_text(:buffer, :effective)}"
+        puts "Inheritable: #{print.caps_text(:buffer, :inheritable)}"
+        puts "Permitted:   #{print.caps_text(:buffer, :permitted)}"
+      end
+    end
+
+    def get_valid_capabilities
+      capabilities = []
+      cap = CapNG::Capability.new
+      cap.each do |_code, capability|
+        capabilities << capability
+      end
+      capabilities
+    end
+
+    def check_capabilities(capabilities, valid_capabilities)
+      capabilities.each do |capability|
+        unless valid_capabilities.include?(capability)
+          raise ArgumentError, "'#{capability}' is not valid capability. Valid Capabilities are:  #{valid_capabilities.join(", ")}"
+        end
+      end
+    end
+
+    def parse_options!(argv)
+      begin
+        rest = @op.parse(argv)
+
+        if rest.length != 0
+          usage nil
+        end
+      rescue
+        usage $!.to_s
+      end
+    end
+  end
+end

data/lib/fluent/command/cat.rb

@@ -0,0 +1,365 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'optparse'
+require 'fluent/env'
+require 'fluent/time'
+require 'fluent/msgpack_factory'
+require 'fluent/version'
+
+op = OptionParser.new
+
+op.banner += " <tag>"
+op.version = Fluent::VERSION
+
+port = 24224
+host = '127.0.0.1'
+unix = false
+socket_path = Fluent::DEFAULT_SOCKET_PATH
+
+config_path = Fluent::DEFAULT_CONFIG_PATH
+format = 'json'
+message_key = 'message'
+time_as_integer = false
+retry_limit = 5
+event_time = nil
+
+op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
+  port = i
+}
+
+op.on('-h', '--host HOST', "fluent host (default: #{host})") {|s|
+  host = s
+}
+
+op.on('-u', '--unix', "use unix socket instead of tcp", TrueClass) {|b|
+  unix = b
+}
+
+op.on('-s', '--socket PATH', "unix socket path (default: #{socket_path})") {|s|
+  socket_path = s
+}
+
+op.on('-f', '--format FORMAT', "input format (default: #{format})") {|s|
+  format = s
+}
+
+op.on('--json', "same as: -f json", TrueClass) {|b|
+  format = 'json'
+}
+
+op.on('--msgpack', "same as: -f msgpack", TrueClass) {|b|
+  format = 'msgpack'
+}
+
+op.on('--none', "same as: -f none", TrueClass) {|b|
+  format = 'none'
+}
+
+op.on('--message-key KEY', "key field for none format (default: #{message_key})") {|s|
+  message_key = s
+}
+
+op.on('--time-as-integer', "Send time as integer for v0.12 or earlier", TrueClass) { |b|
+  time_as_integer = true
+}
+
+op.on('--retry-limit N', "Specify the number of retry limit (default: #{retry_limit})", Integer) {|n|
+  retry_limit = n
+}
+
+op.on('--event-time TIME_STRING', "Specify the time expression string (default: nil)") {|v|
+  event_time = v
+}
+
+singleton_class.module_eval do
+  define_method(:usage) do |msg|
+    puts op.to_s
+    puts "error: #{msg}" if msg
+    exit 1
+  end
+end
+
+begin
+  op.parse!(ARGV)
+
+  if ARGV.length != 1
+    usage nil
+  end
+
+  tag = ARGV.shift
+
+rescue
+  usage $!.to_s
+end
+
+require 'socket'
+require 'yajl'
+require 'msgpack'
+require 'fluent/ext_monitor_require'
+
+class Writer
+  include MonitorMixin
+
+  RetryLimitError = Class.new(StandardError)
+
+  class TimerThread
+    def initialize(writer)
+      @writer = writer
+    end
+
+    def start
+      @finish = false
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      @finish = true
+      @thread.join
+    end
+
+    def run
+      until @finish
+        sleep 1
+        @writer.on_timer
+      end
+    end
+  end
+
+  def initialize(tag, connector, time_as_integer: false, retry_limit: 5, event_time: nil)
+    @tag = tag
+    @connector = connector
+    @socket = false
+
+    @socket_time = Time.now.to_i
+    @socket_ttl = 10  # TODO
+    @error_history = []
+
+    @pending = []
+    @pending_limit = 1024  # TODO
+    @retry_wait = 1
+    @retry_limit = retry_limit
+    @time_as_integer = time_as_integer
+    @event_time = event_time
+
+    super()
+  end
+
+  def secondary_record?(record)
+    record.class != Hash &&
+      record.size == 2 &&
+      record.first.class == Fluent::EventTime &&
+      record.last.class == Hash
+  end
+
+  def write(record)
+    unless secondary_record?(record)
+      if record.class != Hash
+        raise ArgumentError, "Input must be a map (got #{record.class})"
+      end
+    end
+
+    time = if @event_time
+             Fluent::EventTime.parse(@event_time)
+           else
+             Fluent::EventTime.now
+           end
+    time = time.to_i if @time_as_integer
+    entry = if secondary_record?(record)
+              # Even though secondary contains Fluent::EventTime in record,
+              # fluent-cat just ignore it and set Fluent::EventTime.now instead.
+              # This specification is adopted to keep consistency.
+              [time, record.last]
+            else
+              [time, record]
+            end
+    synchronize {
+      unless write_impl([entry])
+        # write failed
+        @pending.push(entry)
+
+        while @pending.size > @pending_limit
+          # exceeds pending limit; trash oldest record
+          time, record = @pending.shift
+          abort_message(time, record)
+        end
+      end
+    }
+  end
+
+  def on_timer
+    now = Time.now.to_i
+
+    synchronize {
+      unless @pending.empty?
+        # flush pending records
+        if write_impl(@pending)
+          # write succeeded
+          @pending.clear
+        end
+      end
+
+      if @socket && @socket_time + @socket_ttl < now
+        # socket is not used @socket_ttl seconds
+        close
+      end
+    }
+  end
+
+  def close
+    @socket.close
+    @socket = nil
+  end
+
+  def start
+    @timer = TimerThread.new(self)
+    @timer.start
+    self
+  end
+
+  def shutdown
+    @timer.shutdown
+  end
+
+  private
+  def write_impl(array)
+    socket = get_socket
+    unless socket
+      return false
+    end
+
+    begin
+      packer = Fluent::MessagePackFactory.packer
+      socket.write packer.pack([@tag, array])
+      socket.flush
+    rescue
+      $stderr.puts "write failed: #{$!}"
+      close
+      return false
+    end
+
+    return true
+  end
+
+  def get_socket
+    unless @socket
+      unless try_connect
+        return nil
+      end
+    end
+
+    @socket_time = Time.now.to_i
+    return @socket
+  end
+
+  def try_connect
+    begin
+      now = Time.now.to_i
+
+      unless @error_history.empty?
+        # wait before re-connecting
+        wait = 1 #@retry_wait * (2 ** (@error_history.size-1))
+        if now <= @socket_time + wait
+          sleep(wait)
+          try_connect
+        end
+      end
+
+      @socket = @connector.call
+      @error_history.clear
+      return true
+
+    rescue RetryLimitError => ex
+      raise ex
+    rescue
+      $stderr.puts "connect failed: #{$!}"
+      @error_history << $!
+      @socket_time = now
+
+      if @retry_limit < @error_history.size
+        # abort all pending records
+        @pending.each {|(time, record)|
+          abort_message(time, record)
+        }
+        @pending.clear
+        @error_history.clear
+        raise RetryLimitError, "exceed retry limit"
+      else
+        retry
+      end
+    end
+  end
+
+  def abort_message(time, record)
+    $stdout.puts "!#{time}:#{Yajl.dump(record)}"
+  end
+end
+
+
+if unix
+  connector = Proc.new {
+    UNIXSocket.open(socket_path)
+  }
+else
+  connector = Proc.new {
+    TCPSocket.new(host, port)
+  }
+end
+
+w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit, event_time: event_time)
+w.start
+
+case format
+when 'json'
+  begin
+    while line = $stdin.gets
+      record = Yajl.load(line)
+      w.write(record)
+    end
+  rescue
+    $stderr.puts $!
+    exit 1
+  end
+
+when 'msgpack'
+  require 'fluent/engine'
+
+  begin
+    u = Fluent::MessagePackFactory.msgpack_unpacker($stdin)
+    u.each {|record|
+      w.write(record)
+    }
+  rescue EOFError
+  rescue
+    $stderr.puts $!
+    exit 1
+  end
+
+when 'none'
+  begin
+    while line = $stdin.gets
+      record = { message_key => line.chomp }
+      w.write(record)
+    end
+  rescue
+    $stderr.puts $!
+    exit 1
+  end
+
+else
+  $stderr.puts "Unknown format '#{format}'"
+  exit 1
+end