fluentd 0.14.10-x86-mingw32 → 0.14.11-x86-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cc9fd1115013c31c251063f81dc0d91be3b5624c
-  data.tar.gz: 95d7e3af0d15244336c42b8d1bfe83f08afd8f36
+  metadata.gz: ef2aa915f245197aeff9a7b048517bb3adbd1421
+  data.tar.gz: e22546a0cb0e5671fa44e13ccc067ebc3c4c6b02
 SHA512:
-  metadata.gz: 92f075a8834ebbcd7b5c1c43433afc8f9d62a53f68e26ef8e82519515dc7abf8b4a9ad03a43307c3a232a84f281a1f3968e2dda160659c54a01da164fe348224
-  data.tar.gz: 5d0f4bc6df7f9bfe63d7c5f9c4b8e8117111bcaa93f60a32a884e90e69523200a458732e452c964edf590af43da71e51789d01a2dff7a9e28c653413909f834a
+  metadata.gz: de6bb3169a143a113303ea17ea97b5c6d1c669f40cc76fe4093657a4fe8d315667c01d5e7abef4f17260967bc182d76f86e7c7e55d46a89aca9a524d071790a4
+  data.tar.gz: 9d69e9d69a5730de61fcc81c828bcc48c32ad9af464f4a329175e043577e662c23508f3fe4f967b6fef43c593d0a4bd6e97e71dd9eafc3123ced6e6fb2ff7ba1

data/.travis.yml

@@ -9,21 +9,29 @@ matrix:
   include:
     - rvm: 2.1.10
       os: linux
-    - rvm: 2.2.5
+    - rvm: 2.2.6
       os: linux
-    - rvm: 2.3.1
+    - rvm: 2.3.3
+      os: linux
+    - rvm: 2.4.0
       os: linux
     - rvm: ruby-head
       os: linux
     - rvm: 2.1.10
       os: osx
-      osx_image: xcode7.3 # OSX 10.11
-    - rvm: 2.2.5
+      osx_image: xcode8.2 # OSX 10.12
+    # - rvm: 2.2.6
+    #   os: osx
+    #   osx_image: xcode8.2 # OSX 10.12
+    # - rvm: 2.3.3
+    #   os: osx
+    #   osx_image: xcode8.2 # OSX 10.12
+    - rvm: 2.4.0
       os: osx
-      osx_image: xcode7.3 # OSX 10.11
+      osx_image: xcode8.2 # OSX 10.12
     - rvm: ruby-head
       os: osx
-      osx_image: xcode 7.3 # OSX 10.11
+      osx_image: xcode 8.2 # OSX 10.12
   allow_failures:
     - rvm: ruby-head
 

data/ChangeLog

@@ -1,5 +1,31 @@
 # v0.14
 
+## Release v0.14.11 - 2016/12/26
+
+### New features / Enhancements
+* Add "root_dir" parameter in <system> directive to configure server root directory, used for buffer/storage paths
+  https://github.com/fluent/fluentd/pull/1374
+* Fix not to restart Fluentd processes when unrecoverable errors occur
+  https://github.com/fluent/fluentd/pull/1359
+* Show warnings in log when output flush operation takes longer time than threshold
+  https://github.com/fluent/fluentd/pull/1370
+* formatter_csv: Raise configuration error when no field names are specified
+  https://github.com/fluent/fluentd/pull/1369
+* in_syslog: Update implementation to use plugin helpers
+  https://github.com/fluent/fluentd/pull/1382
+* in_forward: Add a configuration parameter "source_address_key"
+  https://github.com/fluent/fluentd/pull/1382
+* in_monitor_agent: Add a parameter "include_retry" to get detail retry status
+  https://github.com/fluent/fluentd/pull/1387
+* Add Ruby 2.4 into supported ruby versions
+
+### Bug fixes
+* Fix to set process name of supervisor process
+https://github.com/fluent/fluentd/pull/1380
+* in_forward: Fix a bug not to handle "require_ack_response" correctly
+https://github.com/fluent/fluentd/pull/1389
+
+
 ## Release v0.14.10 - 2016/12/14
 
 ### New features / Enhancement
@@ -29,7 +55,7 @@
 * TimeSlice output plugins (in v0.12 style) raise errors when "utc" parameter is specified
   https://github.com/fluent/fluentd/pull/1319
 * Parser plugins cannot use options for regular expressions
-  https://github.com/fluent/fluentd/pull/1326/files
+  https://github.com/fluent/fluentd/pull/1326
 * Fix bugs not to raise errors to use logger in v0.12 plugins
   https://github.com/fluent/fluentd/pull/1344
   https://github.com/fluent/fluentd/pull/1332
@@ -37,7 +63,7 @@
   https://github.com/fluent/fluentd/pull/1367
 * in_tail: Close files explicitly in tests
   https://github.com/fluent/fluentd/pull/1327
-* in_forward: Fix bug not to convert buffer configurations into v0.14 parameters
+* out_forward: Fix bug not to convert buffer configurations into v0.14 parameters
   https://github.com/fluent/fluentd/pull/1337
 * out_forward: Fix bug to raise error when "expire_dns_cache" is specified
   https://github.com/fluent/fluentd/pull/1346

data/appveyor.yml

@@ -21,6 +21,7 @@ branches:
     - v0.12
     - v0.14
 
+# https://www.appveyor.com/docs/installed-software/#ruby
 environment:
   matrix:
     - ruby_version: "23-x64"

data/lib/fluent/engine.rb

@@ -14,10 +14,6 @@
 #    limitations under the License.
 #
 
-require 'socket'
-
-require 'cool.io'
-
 require 'fluent/config'
 require 'fluent/event'
 require 'fluent/event_router'
@@ -58,8 +54,6 @@ module Fluent
     def init(system_config)
       @system_config = system_config
 
-      BasicSocket.do_not_reverse_lookup = true
-
       suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
       @suppress_config_dump = system_config.suppress_config_dump unless system_config.suppress_config_dump.nil?
       @without_source = system_config.without_source unless system_config.without_source.nil?
@@ -172,6 +166,8 @@ module Fluent
 
     def run
       begin
+        worker_id = ENV['SERVERENGINE_WORKER_ID']
+        $log.info "starting fluentd worker", pid: Process.pid, ppid: Process.ppid, worker: worker_id
         start
 
         if @event_router.match?($log.tag)
@@ -179,6 +175,7 @@ module Fluent
           @log_emit_thread = Thread.new(&method(:log_event_loop))
         end
 
+        $log.info "fluentd worker is now running" # TODO: worker number
         sleep MAINLOOP_SLEEP_INTERVAL until @engine_stopped
 
       rescue Exception => e
@@ -187,7 +184,7 @@ module Fluent
         raise
       end
 
-      $log.info "shutting down fluentd"
+      $log.info "shutting down fluentd worker" # TODO: worker number
       shutdown
       if @log_emit_thread
         @log_event_loop_stop = true

data/lib/fluent/error.rb

@@ -0,0 +1,30 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  class UnrecoverableError < StandardError
+    def initialize(error_message = nil)
+      @message = error_message || "an unrecoverable error occurs in Fluentd process"
+    end
+
+    def to_s
+      @message
+    end
+  end
+
+  class InvalidRootDirectory < UnrecoverableError
+  end
+end

data/lib/fluent/log.rb

@@ -299,8 +299,6 @@ module Fluent
       @out.reset if @out.respond_to?(:reset)
     end
 
-    private
-
     def dump_stacktrace(backtrace, level)
       return if @level > level
 
@@ -433,11 +431,6 @@ module Fluent
       end
     end
 
-    def start
-      @log.reset
-      super
-    end
-
     def terminate
       super
       @log.reset

data/lib/fluent/plugin/base.rb

@@ -32,6 +32,7 @@ module Fluent
         super
         @_state = State.new(false, false, false, false, false, false, false, false, false)
         @_context_router = nil
+        @_fluentd_worker_id = nil
         @under_plugin_development = false
       end
 
@@ -39,6 +40,16 @@ module Fluent
         false
       end
 
+      def plugin_root_dir
+        nil # override this in plugin_id.rb
+      end
+
+      def fluentd_worker_id
+        return @_fluentd_worker_id if @_fluentd_worker_id
+        @_fluentd_worker_id = (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i
+        @_fluentd_worker_id
+      end
+
       def configure(conf)
         super
         @_state ||= State.new(false, false, false, false, false, false, false, false, false)

data/lib/fluent/plugin/buf_file.rb

@@ -32,9 +32,8 @@ module Fluent
 
       DIR_PERMISSION = 0755
 
-      # TODO: buffer_path based on system config
       desc 'The path where buffer chunks are stored.'
-      config_param :path, :string
+      config_param :path, :string, default: nil
 
       config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
       config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
@@ -42,10 +41,6 @@ module Fluent
       config_param :file_permission, :string, default: nil # '0644'
       config_param :dir_permission,  :string, default: nil # '0755'
 
-      ##TODO: Buffer plugin cannot handle symlinks because new API @stage has many writing buffer chunks
-      ##      re-implement this feature on out_file, w/ enqueue_chunk(or generate_chunk) hook + chunk.path
-      # attr_accessor :symlink_path
-
       @@buffer_paths = {}
 
       def initialize
@@ -56,6 +51,14 @@ module Fluent
       def configure(conf)
         super
 
+        unless @path
+          if root_dir = owner.plugin_root_dir
+            @path = File.join(root_dir, 'buffer')
+          else
+            raise Fluent::ConfigError, "buffer path is not configured. specify 'path' in <buffer>"
+          end
+        end
+
         type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
         if @@buffer_paths.has_key?(@path) && !buffer_path_for_test?
           type_using_this_path = @@buffer_paths[@path]
@@ -64,7 +67,6 @@ module Fluent
 
         @@buffer_paths[@path] = type_of_owner
 
-        # TODO: create buffer path with plugin_id, under directory specified by system config
         if File.exist?(@path)
           if File.directory?(@path)
             @path = File.join(@path, 'buffer.*.log')

data/lib/fluent/plugin/formatter_csv.rb

@@ -33,14 +33,16 @@ module Fluent
       def configure(conf)
         super
         @fields = fields.select{|f| !f.empty? }
+        raise ConfigError, "empty value is specified in fields parameter" if @fields.empty?
+
+        @generate_opts = {col_sep: @delimiter, force_quotes: @force_quotes}
       end
 
       def format(tag, time, record)
         row = @fields.map do |key|
           record[key]
         end
-        CSV.generate_line(row, col_sep: @delimiter,
-                          force_quotes: @force_quotes)
+        CSV.generate_line(row, @generate_opts)
       end
     end
   end

data/lib/fluent/plugin/in_forward.rb

@@ -53,6 +53,9 @@ module Fluent::Plugin
     config_param :chunk_size_limit, :size, default: nil
     desc 'Skip an event if incoming event is invalid.'
     config_param :skip_invalid_event, :bool, default: false
+
+    desc "The field name of the client's source address."
+    config_param :source_address_key, :string, default: nil
     desc "The field name of the client's hostname."
     config_param :source_hostname_key, :string, default: nil
 
@@ -98,6 +101,7 @@ module Fluent::Plugin
           raise Fluent::ConfigError, "resolve_hostname must be true with source_hostname_key"
         end
       end
+      @enable_field_injection = @source_address_key || @source_hostname_key
 
       if @security
         if @security.user_auth && @security.users.empty?
@@ -136,7 +140,6 @@ module Fluent::Plugin
             })
         end
       end
-      @lsock = @usock = nil
     end
 
     HEARTBEAT_UDP_PAYLOAD = "\0"
@@ -197,7 +200,7 @@ module Fluent::Plugin
           log.debug "connection established", address: conn.remote_addr, port: conn.remote_port
           state = :established
         when :established
-          options = on_message(msg, chunk_size, conn.remote_host)
+          options = on_message(msg, chunk_size, conn)
           if options && r = response(options)
             log.trace "sent response to fluent socket", address: conn.remote_addr, response: r
             conn.on_write_complete{ conn.close } if @deny_keepalive
@@ -253,7 +256,7 @@ module Fluent::Plugin
       nil
     end
 
-    def on_message(msg, chunk_size, remote_host)
+    def on_message(msg, chunk_size, conn)
       if msg.nil?
         # for future TCP heartbeat_request
         return
@@ -261,7 +264,7 @@ module Fluent::Plugin
 
       # TODO: raise an exception if broken chunk is generated by recoverable situation
       unless msg.is_a?(Array)
-        log.warn "incoming chunk is broken:", host: remote_host, msg: msg
+        log.warn "incoming chunk is broken:", host: conn.remote_host, msg: msg
         return
       end
 
@@ -269,10 +272,10 @@ module Fluent::Plugin
       entries = msg[1]
 
       if @chunk_size_limit && (chunk_size > @chunk_size_limit)
-        log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, host: remote_host, limit: @chunk_size_limit, size: chunk_size
+        log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, host: conn.remote_host, limit: @chunk_size_limit, size: chunk_size
         return
       elsif @chunk_size_warn_limit && (chunk_size > @chunk_size_warn_limit)
-        log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, host: remote_host, limit: @chunk_size_warn_limit, size: chunk_size
+        log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, host: conn.remote_host, limit: @chunk_size_warn_limit, size: chunk_size
       end
 
       case entries
@@ -282,14 +285,16 @@ module Fluent::Plugin
         size = (option && option['size']) || 0
         es_class = (option && option['compressed'] == 'gzip') ? Fluent::CompressedMessagePackEventStream : Fluent::MessagePackEventStream
         es = es_class.new(entries, nil, size.to_i)
-        es = check_and_skip_invalid_event(tag, es, remote_host) if @skip_invalid_event
-        es = add_source_host(es, remote_host) if @source_hostname_key
+        es = check_and_skip_invalid_event(tag, es, conn.remote_host) if @skip_invalid_event
+        if @enable_field_injection
+          es = add_source_info(es, conn)
+        end
         router.emit_stream(tag, es)
 
       when Array
         # Forward
         es = if @skip_invalid_event
-               check_and_skip_invalid_event(tag, entries, remote_host)
+               check_and_skip_invalid_event(tag, entries, conn.remote_host)
              else
                es = Fluent::MultiEventStream.new
                entries.each { |e|
@@ -301,7 +306,9 @@ module Fluent::Plugin
                }
                es
              end
-        es = add_source_host(es, remote_host) if @source_hostname_key
+        if @enable_field_injection
+          es = add_source_info(es, conn)
+        end
         router.emit_stream(tag, es)
         option = msg[2]
 
@@ -310,12 +317,15 @@ module Fluent::Plugin
         time = msg[1]
         record = msg[2]
         if @skip_invalid_event && invalid_event?(tag, time, record)
-          log.warn "got invalid event and drop it:", host: remote_host, tag: tag, time: time, record: record
+          log.warn "got invalid event and drop it:", host: conn.remote_host, tag: tag, time: time, record: record
           return msg[3] # retry never succeeded so return ack and drop incoming event.
         end
         return if record.nil?
         time = Fluent::Engine.now if time.to_i == 0
-        record[@source_hostname_key] = remote_host if @source_hostname_key
+        if @enable_field_injection
+          record[@source_address_key] = conn.remote_addr if @source_address_key
+          record[@source_hostname_key] = conn.remote_host if @source_hostname_key
+        end
         router.emit(tag, time, record)
         option = msg[3]
       end
@@ -340,12 +350,31 @@ module Fluent::Plugin
       new_es
     end
 
-    def add_source_host(es, host)
+    def add_source_info(es, conn)
       new_es = Fluent::MultiEventStream.new
-      es.each { |time, record|
-        record[@source_hostname_key] = host
-        new_es.add(time, record)
-      }
+      if @source_address_key && @source_hostname_key
+        address = conn.remote_addr
+        hostname = conn.remote_host
+        es.each { |time, record|
+          record[@source_address_key] = address
+          record[@source_hostname_key] = hostname
+          new_es.add(time, record)
+        }
+      elsif @source_address_key
+        address = conn.remote_addr
+        es.each { |time, record|
+          record[@source_address_key] = address
+          new_es.add(time, record)
+        }
+      elsif @source_hostname_key
+        hostname = conn.remote_host
+        es.each { |time, record|
+          record[@source_hostname_key] = hostname
+          new_es.add(time, record)
+        }
+      else
+        raise "BUG: don't call this method in this case"
+      end
       new_es
     end
 

data/lib/fluent/plugin/in_http.rb

@@ -36,6 +36,8 @@ module Fluent::Plugin
   class HttpInput < Input
     Fluent::Plugin.register_input('http', self)
 
+    # TODO: update this plugin implementation to use server plugin helper, after adding keepalive feature on it
+
     helpers :parser, :compat_parameters, :event_loop
 
     EMPTY_GIF_IMAGE = "GIF89a\u0001\u0000\u0001\u0000\x80\xFF\u0000\xFF\xFF\xFF\u0000\u0000\u0000,\u0000\u0000\u0000\u0000\u0001\u0000\u0001\u0000\u0000\u0002\u0002D\u0001\u0000;".force_encoding("UTF-8")