fluentd 1.0.2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 599f19a7ae06ad5cfe37bfa47a6b3d5d4c59070c
-  data.tar.gz: 949bcb1356ad507607487ffb072bd73fbce75d85
+  metadata.gz: f7457e2a191772c11ca520188030ea3ef06c211d
+  data.tar.gz: 2ae8a9864a31245d3fca6756cb4f39c82aed53fc
 SHA512:
-  metadata.gz: 714ef7ba99c73efd0f82a57323a829f3c771ad94595bc809406bc43959cd39b6300b82f50d3ecd4292425dc76f55ff4fc9e1207656e0dad4f35c8e33dcc1d941
-  data.tar.gz: 5374ef1b34a077aab0d929256eceb883881bf1b92312bace5b01452cad8c50bd59cc9b5e2d81dfb5461e4057e497f860d07e2a88febd093112668493422a44ed
+  metadata.gz: 6ad103cd892630b1c15452a4749ddba854b2ca73e87bbe252bdf75a7a85964589801e21d002bf6140b23d107dbed207e856a0eaee275ea04c895ebb5df31d361
+  data.tar.gz: 44cfc8efcc83ac52e55fb35afcc93486dbb44ed9ee84ac20fa8da335a011812f04eb256bdb55d741facc8bdbb5e87728d75c11f118fe370f7690b553b082223a

data/.travis.yml

@@ -31,8 +31,14 @@ matrix:
       osx_image: xcode8.3 # OSX 10.12
     - rvm: ruby-head
       os: osx
-      osx_image: xcode 8.3 # OSX 10.12
+      osx_image: xcode8.3 # OSX 10.12
   allow_failures:
+    - rvm: 2.1.10
+      os: osx
+      osx_image: xcode8.3
+    - rvm: 2.4.1
+      os: osx
+      osx_image: xcode8.3
     - rvm: ruby-head
 
 branches:

data/CHANGELOG.md

@@ -1,5 +1,31 @@
 # v1.0
 
+## Release v1.1.0 - 2018/01/17
+
+### New features / Enhancements
+
+* config: Add hostname and worker_id short-cut
+  https://github.com/fluent/fluentd/pull/1814
+* parser_ltsv: Add delimiter_pattern parameter
+  https://github.com/fluent/fluentd/pull/1802
+* record_accessor helper: Support nested field deletion
+  https://github.com/fluent/fluentd/pull/1800
+* record_accessor helper: Expose internal instance `@keys` variable
+  https://github.com/fluent/fluentd/pull/1808
+* log: Improve Log#on_xxx API performance
+  https://github.com/fluent/fluentd/pull/1809
+* time: Improve time formatting performance
+  https://github.com/fluent/fluentd/pull/1796
+* command: Port certificates generating command from secure-forward
+  https://github.com/fluent/fluentd/pull/1818
+
+### Bug fixes
+
+* server helper: Fix TCP + TLS degradation
+  https://github.com/fluent/fluentd/pull/1805
+* time: Fix the method for TimeFormatter#call
+  https://github.com/fluent/fluentd/pull/1813
+
 ## Release v1.0.2 - 2017/12/17
 
 ### New features / Enhancements

data/bin/fluent-ca-generate

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.join(__dir__, 'lib'))
+require 'fluent/command/ca_generate'
+
+Fluent::CaGenerate.new.call

data/fluentd.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", ["~> 1.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
-  gem.add_runtime_dependency("strptime", ["~> 0.1"])
+  gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
   gem.add_runtime_dependency("dig_rb", ["~> 1.0.0"])
 
   # build gem for a certain platform. see also Rakefile

data/lib/fluent/command/ca_generate.rb

@@ -0,0 +1,179 @@
+require 'openssl'
+require 'optparse'
+require 'fileutils'
+
+module Fluent
+  class CaGenerate
+    DEFAULT_OPTIONS = {
+      private_key_length: 2048,
+      cert_country:  'US',
+      cert_state:    'CA',
+      cert_locality: 'Mountain View',
+      cert_common_name: 'Fluentd Forward CA',
+    }
+    HELP_TEXT = <<HELP
+Usage: fluent-ca-genrate DIR_PATH PRIVATE_KEY_PASSPHRASE [--country COUNTRY] [--state STATE] [--locality LOCALITY] [--common-name COMMON_NAME]
+HELP
+
+    def initialize(argv = ARGV)
+      @argv = argv
+      @options = {}
+      @opt_parser = OptionParser.new
+      configure_option_parser
+      @options.merge!(DEFAULT_OPTIONS)
+      parse_options!
+    end
+
+    def usage(msg = nil)
+      puts HELP_TEXT
+      puts "Error: #{msg}" if msg
+      exit 1
+    end
+
+    def call
+      ca_dir, passphrase, = @argv[0..1]
+
+      unless ca_dir && passphrase
+        puts "#{HELP_TEXT}"
+        puts ''
+        exit 1
+      end
+
+      FileUtils.mkdir_p(ca_dir)
+
+      cert, key = Fluent::CaGenerate.generate_ca_pair(@options)
+
+      key_data = key.export(OpenSSL::Cipher.new('aes256'), passphrase)
+      File.open(File.join(ca_dir, 'ca_key.pem'), 'w') do |file|
+        file.write key_data
+      end
+      File.open(File.join(ca_dir, 'ca_cert.pem'), 'w') do |file|
+        file.write cert.to_pem
+      end
+
+      puts "successfully generated: ca_key.pem, ca_cert.pem"
+      puts "copy and use ca_cert.pem to client(out_forward)"
+    end
+
+    def self.certificates_from_file(path)
+      data = File.read(path)
+      pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\n(?:[^-]*\n)+-+END CERTIFICATE-+\n', Regexp::MULTILINE)
+      list = []
+      data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match)}
+      list
+    end
+
+    def self.generate_ca_pair(opts={})
+      key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+      issuer = subject = OpenSSL::X509::Name.new
+      subject.add_entry('C', opts[:cert_country])
+      subject.add_entry('ST', opts[:cert_state])
+      subject.add_entry('L', opts[:cert_locality])
+      subject.add_entry('CN', opts[:cert_common_name])
+
+      digest = OpenSSL::Digest::SHA256.new
+
+      cert = OpenSSL::X509::Certificate.new
+      cert.not_before = Time.at(0)
+      cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after
+      cert.public_key = key
+      cert.serial = 1
+      cert.issuer = issuer
+      cert.subject = subject
+      cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(true)]))
+      cert.sign(key, digest)
+
+      return cert, key
+    end
+
+    def self.generate_server_pair(opts={})
+      key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+      ca_key = OpenSSL::PKey::RSA.new(File.read(opts[:ca_key_path]), opts[:ca_key_passphrase])
+      ca_cert = OpenSSL::X509::Certificate.new(File.read(opts[:ca_cert_path]))
+      issuer = ca_cert.issuer
+
+      subject = OpenSSL::X509::Name.new
+      subject.add_entry('C', opts[:country])
+      subject.add_entry('ST', opts[:state])
+      subject.add_entry('L', opts[:locality])
+      subject.add_entry('CN', opts[:common_name])
+
+      digest = OpenSSL::Digest::SHA256.new
+
+      cert = OpenSSL::X509::Certificate.new
+      cert.not_before = Time.at(0)
+      cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after
+      cert.public_key = key
+      cert.serial = 2
+      cert.issuer = issuer
+      cert.subject = subject
+
+      cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
+      cert.add_extension OpenSSL::X509::Extension.new('nsCertType', 'server')
+
+      cert.sign ca_key, digest
+
+      return cert, key
+    end
+
+    def self.generate_self_signed_server_pair(opts={})
+      key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+      issuer = subject = OpenSSL::X509::Name.new
+      subject.add_entry('C', opts[:country])
+      subject.add_entry('ST', opts[:state])
+      subject.add_entry('L', opts[:locality])
+      subject.add_entry('CN', opts[:common_name])
+
+      digest = OpenSSL::Digest::SHA256.new
+
+      cert = OpenSSL::X509::Certificate.new
+      cert.not_before = Time.at(0)
+      cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after
+      cert.public_key = key
+      cert.serial = 1
+      cert.issuer = issuer
+      cert.subject  = subject
+      cert.sign(key, digest)
+
+      return cert, key
+    end
+
+    private
+
+    def configure_option_parser
+      @opt_parser.banner = HELP_TEXT
+
+      @opt_parser.on('--key-length [KEY_LENGTH]',
+                     "configure key length. (default: #{DEFAULT_OPTIONS[:private_key_length]})") do |v|
+        @options[:private_key_length] = v.to_i
+      end
+
+      @opt_parser.on('--country [COUNTRY]',
+                     "configure country. (default: #{DEFAULT_OPTIONS[:cert_country]})") do |v|
+        @options[:cert_country] = v.upcase
+      end
+
+      @opt_parser.on('--state [STATE]',
+                     "configure state. (default: #{DEFAULT_OPTIONS[:cert_state]})") do |v|
+        @options[:cert_state] = v
+      end
+
+      @opt_parser.on('--locality [LOCALITY]',
+                     "configure locality. (default: #{DEFAULT_OPTIONS[:cert_locality]})") do |v|
+        @options[:cert_locality] = v
+      end
+
+      @opt_parser.on('--common-name [COMMON_NAME]',
+                     "configure common name (default: #{DEFAULT_OPTIONS[:cert_common_name]})") do |v|
+        @options[:cert_common_name] = v
+      end
+    end
+
+    def parse_options!
+      @opt_parser.parse!(@argv)
+    end
+  end
+end

data/lib/fluent/config/literal_parser.rb

@@ -18,6 +18,7 @@ require 'stringio'
 
 require 'json'
 require 'yajl'
+require 'socket'
 require 'irb/ruby-lex'  # RubyLex
 
 require 'fluent/config/basic_parser'
@@ -162,6 +163,11 @@ module Fluent
         if @eval_context.nil?
           parse_error! "embedded code is not allowed in this file"
         end
+        # Add hostname and worker_id to code for preventing unused warnings
+        code = <<EOM + code
+hostname = Socket.gethostname
+worker_id = ENV['SERVERENGINE_WORKER_ID'] || ''
+EOM
         @eval_context.instance_eval(code)
       end
 

data/lib/fluent/log.rb

@@ -163,10 +163,12 @@ module Fluent
     def format=(fmt)
       return if @format == fmt
 
+      @time_format = '%Y-%m-%d %H:%M:%S %z'
+      @time_formatter = Strftime.new(@time_format) rescue nil
+
       case fmt
       when :text
         @format = :text
-        @time_format = '%Y-%m-%d %H:%M:%S %z'
         @formatter = Proc.new { |type, time, level, msg|
           r = caller_line(type, time, @depth_offset, level)
           r << msg
@@ -174,10 +176,9 @@ module Fluent
         }
       when :json
         @format = :json
-        @time_format = '%Y-%m-%d %H:%M:%S %z'
         @formatter = Proc.new { |type, time, level, msg|
           r = {
-            'time' => time.strftime(@time_format),
+            'time' => format_time(time),
             'level' => LEVEL_TEXT[level],
             'message' => msg
           }
@@ -191,6 +192,11 @@ module Fluent
       nil
     end
 
+    def time_format=(time_fmt)
+      @time_format = time_fmt
+      @time_formatter = Strftime.new(@time_format) rescue nil
+    end
+
     def reopen!
       # do nothing in @logger.reopen! because it's already reopened in Supervisor.load_config
       @logger.reopen! if @logger
@@ -261,9 +267,9 @@ module Fluent
       end
     end
 
-    def on_trace(&block)
+    def on_trace
       return if @level > LEVEL_TRACE
-      block.call if block
+      yield
     end
 
     def trace(*args, &block)
@@ -282,9 +288,9 @@ module Fluent
       dump_stacktrace(type, backtrace, LEVEL_TRACE)
     end
 
-    def on_debug(&block)
+    def on_debug
       return if @level > LEVEL_DEBUG
-      block.call if block
+      yield
     end
 
     def debug(*args, &block)
@@ -302,9 +308,9 @@ module Fluent
       dump_stacktrace(type, backtrace, LEVEL_DEBUG)
     end
 
-    def on_info(&block)
+    def on_info
       return if @level > LEVEL_INFO
-      block.call if block
+      yield
     end
 
     def info(*args, &block)
@@ -322,9 +328,9 @@ module Fluent
       dump_stacktrace(type, backtrace, LEVEL_INFO)
     end
 
-    def on_warn(&block)
+    def on_warn
       return if @level > LEVEL_WARN
-      block.call if block
+      yield
     end
 
     def warn(*args, &block)
@@ -342,9 +348,9 @@ module Fluent
       dump_stacktrace(type, backtrace, LEVEL_WARN)
     end
 
-    def on_error(&block)
+    def on_error
       return if @level > LEVEL_ERROR
-      block.call if block
+      yield
     end
 
     def error(*args, &block)
@@ -362,9 +368,9 @@ module Fluent
       dump_stacktrace(type, backtrace, LEVEL_ERROR)
     end
 
-    def on_fatal(&block)
+    def on_fatal
       return if @level > LEVEL_FATAL
-      block.call if block
+      yield
     end
 
     def fatal(*args, &block)
@@ -423,7 +429,7 @@ module Fluent
         end
       else
         r = {
-          'time' => time.strftime(@time_format),
+          'time' => format_time(time),
           'level' => LEVEL_TEXT[level],
         }
         if wid = get_worker_id(type)
@@ -491,7 +497,7 @@ module Fluent
                        else
                          "".freeze
                        end
-      log_msg = "#{time.strftime(@time_format)} [#{LEVEL_TEXT[level]}]: #{worker_id_part}"
+      log_msg = "#{format_time(time)} [#{LEVEL_TEXT[level]}]: #{worker_id_part}"
       if @debug_mode
         line = caller(depth+1)[0]
         if match = /^(.+?):(\d+)(?::in `(.*)')?/.match(line)
@@ -503,6 +509,10 @@ module Fluent
       end
       return log_msg
     end
+
+    def format_time(time)
+      @time_formatter ? @time_formatter.exec(time) : time.strftime(@time_format)
+    end
   end
 
 
@@ -545,8 +555,8 @@ module Fluent
     extend Forwardable
     def_delegators '@logger', :get_worker_id, :enable_color?, :enable_debug, :enable_event,
       :disable_events, :log_event_enabled, :log_event_enamed=, :time_format, :time_format=,
-      :event, :caller_line, :puts, :write, :<<, :flush, :reset, :out, :out=,
-      :optional_header, :optional_header=, :optional_attrs, :optional_attrs=
+      :time_formatter, :time_formatter=, :event, :caller_line, :puts, :write, :<<, :flush,
+      :reset, :out, :out=, :optional_header, :optional_header=, :optional_attrs, :optional_attrs=
   end
 
 

data/lib/fluent/plugin/buffer.rb

@@ -240,7 +240,8 @@ module Fluent
       end
 
       def add_metadata(metadata)
-        log.trace "adding metadata", instance: self.object_id, metadata: metadata
+        log.on_trace { log.trace "adding metadata", instance: self.object_id, metadata: metadata }
+
         synchronize do
           if i = @metadata_list.index(metadata)
             @metadata_list[i]
@@ -263,7 +264,7 @@ module Fluent
         return if metadata_and_data.size < 1
         raise BufferOverflowError, "buffer space has too many data" unless storable?
 
-        log.trace "writing events into buffer", instance: self.object_id, metadata_size: metadata_and_data.size
+        log.on_trace { log.trace "writing events into buffer", instance: self.object_id, metadata_size: metadata_and_data.size }
 
         staged_bytesize = 0
         operated_chunks = []
@@ -382,7 +383,8 @@ module Fluent
       end
 
       def enqueue_chunk(metadata)
-        log.trace "enqueueing chunk", instance: self.object_id, metadata: metadata
+        log.on_trace { log.trace "enqueueing chunk", instance: self.object_id, metadata: metadata }
+
         chunk = synchronize do
           @stage.delete(metadata)
         end
@@ -406,7 +408,8 @@ module Fluent
       end
 
       def enqueue_unstaged_chunk(chunk)
-        log.trace "enqueueing unstaged chunk", instance: self.object_id, metadata: chunk.metadata
+        log.on_trace { log.trace "enqueueing unstaged chunk", instance: self.object_id, metadata: chunk.metadata }
+
         synchronize do
           chunk.synchronize do
             metadata = chunk.metadata
@@ -419,7 +422,8 @@ module Fluent
       end
 
       def enqueue_all
-        log.trace "enqueueing all chunks in buffer", instance: self.object_id
+        log.on_trace { log.trace "enqueueing all chunks in buffer", instance: self.object_id }
+
         if block_given?
           synchronize{ @stage.keys }.each do |metadata|
             # NOTE: The following line might cause data race depending on Ruby implementations except CRuby
@@ -438,7 +442,8 @@ module Fluent
 
       def dequeue_chunk
         return nil if @queue.empty?
-        log.trace "dequeueing a chunk", instance: self.object_id
+        log.on_trace { log.trace "dequeueing a chunk", instance: self.object_id }
+
         synchronize do
           chunk = @queue.shift
 
@@ -453,7 +458,8 @@ module Fluent
       end
 
       def takeback_chunk(chunk_id)
-        log.trace "taking back a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id)
+        log.on_trace { log.trace "taking back a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id) }
+
         synchronize do
           chunk = @dequeued.delete(chunk_id)
           return false unless chunk # already purged by other thread
@@ -470,7 +476,8 @@ module Fluent
           return nil unless chunk # purged by other threads
 
           metadata = chunk.metadata
-          log.trace "purging a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
+          log.on_trace { log.trace "purging a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata }
+
           begin
             bytesize = chunk.bytesize
             chunk.purge
@@ -489,7 +496,8 @@ module Fluent
       end
 
       def clear_queue!
-        log.trace "clearing queue", instance: self.object_id
+        log.on_trace { log.trace "clearing queue", instance: self.object_id }
+
         synchronize do
           until @queue.empty?
             begin

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -27,6 +27,8 @@ module Fluent::Plugin
   class RecordTransformerFilter < Fluent::Plugin::Filter
     Fluent::Plugin.register_filter('record_transformer', self)
 
+    helpers :record_accessor
+
     desc 'A comma-delimited list of keys to delete.'
     config_param :remove_keys, :array, default: nil
     desc 'A comma-delimited list of keys to keep.'
@@ -56,6 +58,10 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "`renew_record` must be true to use `keep_keys`" unless @renew_record
       end
 
+      @key_deleters = if @remove_keys
+                        @remove_keys.map { |k| record_accessor_create(k) }
+                      end
+
       placeholder_expander_params = {
         log: log,
         auto_typecast: @auto_typecast,
@@ -96,8 +102,7 @@ module Fluent::Plugin
           if @renew_time_key && new_record.has_key?(@renew_time_key)
             time = Fluent::EventTime.from_time(Time.at(new_record[@renew_time_key].to_f))
           end
-          @remove_keys.each {|k| new_record.delete(k) } if @remove_keys
-
+          @key_deleters.each { |deleter| deleter.delete(new_record) } if @key_deleters
           new_es.add(time, new_record)
         rescue => e
           router.emit_error_event(tag, time, record, e)

data/lib/fluent/plugin/formatter_stdout.rb

@@ -21,13 +21,14 @@ module Fluent
     class StdoutFormatter < Formatter
       Plugin.register_formatter('stdout', self)
 
-      TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%9N %z'
+      TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%N %z'
 
       config_param :output_type, :string, default: 'json'
 
       def configure(conf)
         super
 
+        @time_formatter = Strftime.new(TIME_FORMAT)
         @sub_formatter = Plugin.new_formatter(@output_type, parent: self.owner)
         @sub_formatter.configure(conf)
       end
@@ -38,7 +39,7 @@ module Fluent
       end
 
       def format(tag, time, record)
-        "#{Time.at(time).localtime.strftime(TIME_FORMAT)} #{tag}: #{@sub_formatter.format(tag, time, record).chomp}\n"
+        "#{@time_formatter.exec(Time.at(time).localtime)} #{tag}: #{@sub_formatter.format(tag, time, record).chomp}\n"
       end
 
       def stop

data/lib/fluent/plugin/out_stdout.rb

@@ -24,7 +24,6 @@ module Fluent::Plugin
 
     DEFAULT_LINE_FORMAT_TYPE = 'stdout'
     DEFAULT_FORMAT_TYPE = 'json'
-    TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%9N %z'
 
     config_section :buffer do
       config_set_default :chunk_keys, ['tag']

data/lib/fluent/plugin/output.rb

@@ -971,7 +971,8 @@ module Fluent
       end
 
       def commit_write(chunk_id, delayed: @delayed_commit, secondary: false)
-        log.trace "committing write operation to a chunk", chunk: dump_unique_id_hex(chunk_id), delayed: delayed
+        log.on_trace { log.trace "committing write operation to a chunk", chunk: dump_unique_id_hex(chunk_id), delayed: delayed }
+
         if delayed
           @dequeued_chunks_mutex.synchronize do
             @dequeued_chunks.delete_if{ |info| info.chunk_id == chunk_id }
@@ -1057,7 +1058,7 @@ module Fluent
         chunk = @buffer.dequeue_chunk
         return unless chunk
 
-        log.trace "trying flush for a chunk", chunk: dump_unique_id_hex(chunk.unique_id)
+        log.on_trace { log.trace "trying flush for a chunk", chunk: dump_unique_id_hex(chunk.unique_id) }
 
         output = self
         using_secondary = false

data/lib/fluent/plugin/parser_ltsv.rb

@@ -23,11 +23,20 @@ module Fluent
 
       desc 'The delimiter character (or string) of TSV values'
       config_param :delimiter, :string, default: "\t"
+      desc 'The delimiter pattern of TSV values'
+      config_param :delimiter_pattern, default: nil do |value|
+        Regexp.compile(value[1..-2]) if value
+      end
       desc 'The delimiter character between field name and value'
       config_param :label_delimiter, :string, default: ":"
 
       config_set_default :time_key, 'time'
 
+      def configure(conf)
+        super
+        @delimiter = @delimiter_pattern || @delimiter
+      end
+
       def parse(text)
         r = {}
         text.split(@delimiter).each do |pair|

data/lib/fluent/plugin_helper/record_accessor.rb

@@ -32,13 +32,25 @@ module Fluent
       end
 
       class Accessor
+        attr_reader :keys
+
         def initialize(param)
           @keys = Accessor.parse_parameter(param)
 
-          # Call [] for single key to reduce dig overhead
-          m = method(@keys.is_a?(Array) ? :call_dig : :call_index)
+          if @keys.is_a?(Array)
+            @last_key = @keys.last
+            @dig_keys = @keys[0..-2]
+            mcall = method(:call_dig)
+            mdelete = method(:delete_nest)
+          else
+            # Call [] for single key to reduce dig overhead
+            mcall = method(:call_index)
+            mdelete = method(:delete_top)
+          end
+
           singleton_class.module_eval do
-            define_method(:call, m)
+            define_method(:call, mcall)
+            define_method(:delete, mdelete)
           end
         end
 
@@ -55,6 +67,25 @@ module Fluent
           r[@keys]
         end
 
+        def delete(r)
+        end
+
+        def delete_nest(r)
+          if target = r.dig(*@dig_keys)
+            if target.is_a?(Array)
+              target.delete_at(@last_key)
+            else
+              target.delete(@last_key)
+            end
+          end
+        rescue
+          nil
+        end
+
+        def delete_top(r)
+          r.delete(@keys)
+        end
+
         def self.parse_parameter(param)
           if param.start_with?('$.')
             parse_dot_notation(param)

data/lib/fluent/plugin_helper/server.rb

@@ -446,9 +446,12 @@ module Fluent
       class TLSCallbackSocket < CallbackSocket
         ENABLED_EVENTS = [:data, :write_complete, :close]
 
+        attr_accessor :buffer
+
         def initialize(sock)
           super("tls", sock, ENABLED_EVENTS)
           @peeraddr = (@sock.to_io.peeraddr rescue PEERADDR_FAILED)
+          @buffer = ''
         end
 
         def write(data)

data/lib/fluent/time.rb

@@ -319,21 +319,24 @@ module Fluent
       @tc2 = 0
       @tc2_str = nil
 
+      strftime = format && (Strftime.new(format) rescue nil)
       if format && format =~ /(^|[^%])(%%)*%L|(^|[^%])(%%)*%\d*N/
         define_singleton_method(:format, method(:format_with_subsec))
         define_singleton_method(:call, method(:format_with_subsec))
       else
         define_singleton_method(:format, method(:format_without_subsec))
-        define_singleton_method(:call, method(:format_with_subsec))
+        define_singleton_method(:call, method(:format_without_subsec))
       end
 
-      formatter = Fluent::Timezone.formatter(timezone, format)
+      formatter = Fluent::Timezone.formatter(timezone, strftime ? strftime : format)
       @format_nocache = case
-                        when formatter           then formatter
-                        when format && localtime then ->(time){ Time.at(time).strftime(format) }
-                        when format              then ->(time){ Time.at(time).utc.strftime(format) }
-                        when localtime           then ->(time){ Time.at(time).iso8601 }
-                        else                          ->(time){ Time.at(time).utc.iso8601 }
+                        when formatter             then formatter
+                        when strftime && localtime then ->(time){ strftime.exec(Time.at(time)) }
+                        when format && localtime   then ->(time){ Time.at(time).strftime(format) }
+                        when strftime              then ->(time){ strftime.exec(Time.at(time).utc) }
+                        when format                then ->(time){ Time.at(time).utc.strftime(format) }
+                        when localtime             then ->(time){ Time.at(time).iso8601 }
+                        else                            ->(time){ Time.at(time).utc.iso8601 }
                         end
     end
 

data/lib/fluent/timezone.rb

@@ -97,10 +97,15 @@ module Fluent
       if NUMERIC_PATTERN === timezone
         offset = Time.zone_offset(timezone)
 
-        if format
+        case
+        when format.is_a?(String)
           return Proc.new {|time|
             Time.at(time).localtime(offset).strftime(format)
           }
+        when format.is_a?(Strftime)
+          return Proc.new {|time|
+            format.exec(Time.at(time).localtime(offset))
+          }
         else
           return Proc.new {|time|
             Time.at(time).localtime(offset).iso8601
@@ -116,10 +121,15 @@ module Fluent
           return nil
         end
 
-        if format
+        case
+        when format.is_a?(String)
           return Proc.new {|time|
             Time.at(time).localtime(tz.period_for_utc(time).utc_total_offset).strftime(format)
           }
+        when format.is_a?(Strftime)
+          return Proc.new {|time|
+            format.exec(Time.at(time).localtime(tz.period_for_utc(time).utc_total_offset))
+          }
         else
           return Proc.new {|time|
             Time.at(time).localtime(tz.period_for_utc(time).utc_total_offset).iso8601

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.0.2'
+  VERSION = '1.1.0'
 
 end

data/test/command/test_ca_generate.rb

@@ -0,0 +1,70 @@
+require_relative '../helper'
+
+require 'yajl'
+require 'flexmock/test_unit'
+require 'tmpdir'
+
+require 'fluent/command/ca_generate'
+require 'fluent/event'
+
+class TestFluentCaGenerate < ::Test::Unit::TestCase
+  def test_generate_ca_pair
+    cert, key = Fluent::CaGenerate.generate_ca_pair(Fluent::CaGenerate::DEFAULT_OPTIONS)
+    assert_equal(OpenSSL::X509::Certificate, cert.class)
+    assert_true(key.private?)
+  end
+
+  def test_ca_generate
+    dumped_output = capture_stdout do
+      Dir.mktmpdir do |dir|
+        Fluent::CaGenerate.new([dir, "fluentd"]).call
+        assert_true(File.exist?(File.join(dir, "ca_key.pem")))
+        assert_true(File.exist?(File.join(dir, "ca_cert.pem")))
+      end
+    end
+    expected = <<TEXT
+successfully generated: ca_key.pem, ca_cert.pem
+copy and use ca_cert.pem to client(out_forward)
+TEXT
+    assert_equal(expected, dumped_output)
+  end
+
+  sub_test_case "configure options" do
+    test "should respond multiple options" do
+      dumped_output = capture_stdout do
+        Dir.mktmpdir do |dir|
+          Fluent::CaGenerate.new([dir, "fluentd",
+                                  "--country", "JP", "--key-length", "4096",
+                                  "--state", "Tokyo", "--locality", "Chiyoda-ku",
+                                  "--common-name", "Forward CA"]).call
+          assert_true(File.exist?(File.join(dir, "ca_key.pem")))
+          assert_true(File.exist?(File.join(dir, "ca_cert.pem")))
+        end
+      end
+      expected = <<TEXT
+successfully generated: ca_key.pem, ca_cert.pem
+copy and use ca_cert.pem to client(out_forward)
+TEXT
+      assert_equal(expected, dumped_output)
+    end
+
+    test "invalid options" do
+      Dir.mktmpdir do |dir|
+        assert_raise(OptionParser::InvalidOption) do
+          Fluent::CaGenerate.new([dir, "fluentd",
+                                  "--invalid"]).call
+        end
+        assert_false(File.exist?(File.join(dir, "ca_key.pem")))
+        assert_false(File.exist?(File.join(dir, "ca_cert.pem")))
+      end
+    end
+
+    test "empty options" do
+      assert_raise(SystemExit) do
+        capture_stdout do
+          Fluent::CaGenerate.new([]).call
+        end
+      end
+    end
+  end
+end

data/test/command/test_fluentd.rb

@@ -494,14 +494,14 @@ CONF
     end
 
     test 'success to start the number of workers specified in configuration' do
-      conf = <<CONF
+      conf = <<'CONF'
 <system>
   workers 2
   root_dir #{@root_path}
 </system>
 <source>
   @type dummy
-  @id dummy
+  @id "dummy#{worker_id}" # check worker_id works or not with actual command
   @label @dummydata
   tag dummy
   dummy {"message": "yay!"}

data/test/config/test_literal_parser.rb

@@ -232,6 +232,14 @@ module Fluent::Config
       test("\"\#{\n=begin\n}\"") { assert_parse_error("\"\#{\n=begin\n}\"") }  # error in embedded ruby code
       test('"#{v1}foo#{v2}"') { assert_text_parsed_as("#{v1}foo#{v2}", '"#{v1}foo#{v2}"') }
       test('"#{1+1}foo#{2+2}bar"') { assert_text_parsed_as("#{1+1}foo#{2+2}bar", '"#{1+1}foo#{2+2}bar"') }
+      test('"foo#{hostname}"') { assert_text_parsed_as("foo#{Socket.gethostname}", '"foo#{hostname}"') }
+      test('"foo#{worker_id}"') {
+        ENV.delete('SERVERENGINE_WORKER_ID')
+        assert_text_parsed_as("foo", '"foo#{worker_id}"')
+        ENV['SERVERENGINE_WORKER_ID'] = '1'
+        assert_text_parsed_as("foo1", '"foo#{worker_id}"')
+        ENV.delete('SERVERENGINE_WORKER_ID')
+      }
     end
 
     sub_test_case 'array parsing' do

data/test/plugin/test_filter_record_transformer.rb

@@ -44,7 +44,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       d = create_driver(config)
       d.run {
         msgs.each { |msg|
-          d.feed(@tag, @time, {'foo' => 'bar', 'message' => msg})
+          d.feed(@tag, @time, {'foo' => 'bar', 'message' => msg, 'nest' => {'k1' => 'v1', 'k2' => 'v2'}})
         }
       }
       d.filtered
@@ -68,6 +68,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         assert_equal(@tag, r['tag'])
         assert_equal(Time.at(@time).localtime.to_s, r['time'])
         assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", r['message'])
+        assert_equal({'k1' => 'v1', 'k2' => 'v2'}, r['nest'])
       end
     end
 
@@ -83,6 +84,14 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       end
     end
 
+    test 'remove_keys with nested key' do
+      config = CONFIG + %[remove_keys $.nest.k1]
+      filtered = filter(config)
+      filtered.each_with_index do |(_t, r), i|
+        assert_not_include(r['nest'], 'k1')
+      end
+    end
+
     test 'renew_record' do
       config = CONFIG + %[renew_record true]
       msgs = ['1', '2']

data/test/plugin/test_parser_labeled_tsv.rb

@@ -125,4 +125,21 @@ class LabeledTSVParserTest < ::Test::Unit::TestCase
       assert_equal record['b'], ' '
     end
   end
+
+  data("single space" => ["k1=v1 k2=v2", { "k1" => "v1", "k2" => "v2" }],
+       "multiple space" => ["k1=v1    k2=v2", { "k1" => "v1", "k2" => "v2" }],
+       "reverse" => ["k2=v2 k1=v1", { "k1" => "v1", "k2" => "v2" }],
+       "tab" => ["k2=v2\tk1=v1", { "k1" => "v1", "k2" => "v2" }],
+       "tab and space" => ["k2=v2\t k1=v1", { "k1" => "v1", "k2" => "v2" }])
+  def test_parse_with_delimiter_pattern(data)
+    text, expected = data
+    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
+    parser.configure(
+                     'delimiter_pattern' => '/\s+/',
+                     'label_delimiter' => '='
+                    )
+    parser.instance.parse(text) do |_time, record|
+      assert_equal(expected, record)
+    end
+  end
 end

data/test/plugin_helper/test_record_accessor.rb

@@ -62,6 +62,40 @@ class RecordAccessorHelperTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case 'attr_reader :keys' do
+    setup do
+      @d = Dummy.new
+    end
+
+    data('normal' => 'key1',
+         'space' => 'ke y2',
+         'dot key' => 'this.is.key3')
+    test 'access single key' do |param|
+      accessor = @d.record_accessor_create(param)
+      assert_equal param, accessor.keys
+    end
+
+    test "nested bracket keys with dot" do
+      accessor = @d.record_accessor_create("$['key1']['this.is.key3']")
+      assert_equal ['key1','this.is.key3'], accessor.keys
+    end
+
+    data('dot' => '$.key1.key2[0]',
+         'bracket' => "$['key1']['key2'][0]",
+         'bracket w/ double quotes' => '$["key1"]["key2"][0]')
+    test "nested keys ['key1', 'key2', 0]" do |param|
+      accessor = @d.record_accessor_create(param)
+      assert_equal ['key1', 'key2', 0], accessor.keys
+    end
+
+    data('bracket' => "$['key1'][0]['ke y2']",
+         'bracket w/ double quotes' => '$["key1"][0]["ke y2"]')
+    test "nested keys ['key1', 0, 'ke y2']" do |param|
+      accessor = @d.record_accessor_create(param)
+      assert_equal ['key1', 0, 'ke y2'], accessor.keys
+    end
+  end
+
   sub_test_case Fluent::PluginHelper::RecordAccessor::Accessor do
     setup do
       @d = Dummy.new
@@ -114,4 +148,37 @@ class RecordAccessorHelperTest < Test::Unit::TestCase
       end
     end
   end
+
+  sub_test_case 'Fluent::PluginHelper::RecordAccessor::Accessor#delete' do
+    setup do
+      @d = Dummy.new
+    end
+
+    data('normal' => 'key1',
+         'space' => 'ke y2',
+         'dot key' => 'this.is.key3')
+    test 'delete top key' do |param|
+      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}
+      accessor = @d.record_accessor_create(param)
+      accessor.delete(r)
+      assert_not_include(r, param)
+    end
+
+    data('bracket' => "$['key1'][0]['ke y2']",
+         'bracket w/ double quotes' => '$["key1"][0]["ke y2"]')
+    test "delete nested keys ['key1', 0, 'ke y2']" do |param|
+      r = {'key1' => [{'ke y2' => "value"}]}
+      accessor = @d.record_accessor_create(param)
+      accessor.delete(r)
+      assert_not_include(r['key1'][0], 'ke y2')
+    end
+
+    test "don't raise an error when unexpected record is coming" do
+      r = {'key1' => [{'key3' => "value"}]}
+      accessor = @d.record_accessor_create("$['key1']['key2']['key3']")
+      assert_nothing_raised do
+        assert_nil accessor.delete(r)
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.1.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-18 00:00:00.000000000 Z
+date: 2018-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -150,16 +150,22 @@ dependencies:
   name: strptime
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: dig_rb
   requirement: !ruby/object:Gem::Requirement
@@ -306,6 +312,7 @@ email:
 - frsyuki@gmail.com
 executables:
 - fluent-binlog-reader
+- fluent-ca-generate
 - fluent-cat
 - fluent-debug
 - fluent-gem
@@ -332,6 +339,7 @@ files:
 - Vagrantfile
 - appveyor.yml
 - bin/fluent-binlog-reader
+- bin/fluent-ca-generate
 - bin/fluent-cat
 - bin/fluent-debug
 - bin/fluent-gem
@@ -379,6 +387,7 @@ files:
 - lib/fluent/clock.rb
 - lib/fluent/command/binlog_reader.rb
 - lib/fluent/command/bundler_injection.rb
+- lib/fluent/command/ca_generate.rb
 - lib/fluent/command/cat.rb
 - lib/fluent/command/debug.rb
 - lib/fluent/command/fluentd.rb
@@ -580,6 +589,7 @@ files:
 - templates/plugin_config_formatter/param.md.erb
 - templates/plugin_config_formatter/section.md.erb
 - test/command/test_binlog_reader.rb
+- test/command/test_ca_generate.rb
 - test/command/test_fluentd.rb
 - test/command/test_plugin_config_formatter.rb
 - test/command/test_plugin_generator.rb
@@ -748,6 +758,7 @@ specification_version: 4
 summary: Fluentd event collector
 test_files:
 - test/command/test_binlog_reader.rb
+- test/command/test_ca_generate.rb
 - test/command/test_fluentd.rb
 - test/command/test_plugin_config_formatter.rb
 - test/command/test_plugin_generator.rb