fluentd 0.14.14 → 0.14.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 55d7fb0ec7d40340b7c151ecd199d111ce6900d8
-  data.tar.gz: 69a6ca966e185eb804a882e18ddd4888eba7f468
+  metadata.gz: 3f2a4f15bf32737020c69883079694ad81648cff
+  data.tar.gz: 142851ddfb0483e49bc1b71aed6e7e98db295c34
 SHA512:
-  metadata.gz: 953c3799e0101162f11710cf798a2cae19257430ae4a6086b8029dd95c6d8b1ee003bcf9d47a15b18a81c46b591d02b08f2be053d0a403d5d249fc9d2dc37100
-  data.tar.gz: e9d9297f25744020754a78ad9fc552a994eeec9798948e007a1bb290b99438645a7b954ab7389043b4a0cf011aa068385b6f6e7c0a53b46d204ad037b875bc90
+  metadata.gz: ad5bcad4d26b4cfaa59b7b59c5100cfe0a651206dbadfb0b519369f8b095951bf68f45c8df67dd6954bdb8fc64a714cd4911063f9c5eebb9d963ebd7dd71af4f
+  data.tar.gz: c38a2498db6ea742f50103beca1e9c4f8e19fd1f811e84152a2d7679d0225edb65f1cb7e75975f414ec0ea5343a4ca2f2342f233b55bcd4969bfcf481260ec26

data/ChangeLog

@@ -1,5 +1,33 @@
 # v0.14
 
+## Release v0.14.15 - 2017/04/23
+
+### New features / Enhancements
+
+* Add <worker N> directive
+  https://github.com/fluent/fluentd/pull/1507
+* in_tail: Do not warn that directories are unreadable in the in_tail plugin
+  https://github.com/fluent/fluentd/pull/1540
+* output: Add formatted_to_msgpack_binary? to Output plugin API
+  https://github.com/fluent/fluentd/pull/1547
+* windows: Allow the Windows Service name Fluentd runs as to be configurable
+  https://github.com/fluent/fluentd/pull/1548
+
+### Bug fixes
+
+* in_http: Fix X-Forwarded-For header handling. Accpet multiple headers
+  https://github.com/fluent/fluentd/pull/1535
+* Fix backward compatibility with Fluent::DetachProcess and Fluent::DetachMultiProcess
+  https://github.com/fluent/fluentd/pull/1522
+* fix typo
+  https://github.com/fluent/fluentd/pull/1521
+  https://github.com/fluent/fluentd/pull/1523
+  https://github.com/fluent/fluentd/pull/1544
+* test: Fix out_file test with timezone
+  https://github.com/fluent/fluentd/pull/1546
+* windows: Quote the file path to the Ruby bin directory when starting fluentd as a windows service
+  https://github.com/fluent/fluentd/pull/1536
+
 ## Release v0.14.14 - 2017/03/23
 
 ### New features / Enhancements

data/example/worker_section.conf

@@ -0,0 +1,36 @@
+<system>
+  workers 4
+  root_dir /path/fluentd/root
+</system>
+
+<source> # top-level sections works on all workers in parallel
+  @type forward
+  port 24224
+</source>
+
+<match all> # this sections also works on all workers in parallel
+  @type stdout
+  <inject>
+    worker_id_key worker_id
+  </inject>
+</match>
+
+<worker 0> # this section works only on first worker process
+  <source>
+    @type tail
+    format none
+    path /var/log/fluentd_test.log
+    pos_file /var/log/fluentd_test.pos
+    tag tail
+    rotate_wait 5
+    read_from_head true
+    refresh_interval 60
+  </source>
+
+  <match tail>
+    @type stdout
+    <inject>
+      worker_id_key worker_id
+    </inject>
+  </match>
+</worker>

data/lib/fluent/agent.rb

@@ -62,6 +62,7 @@ module Fluent
 
       # initialize <match> and <filter> elements
       conf.elements('filter', 'match').each { |e|
+        next if e.for_another_worker?
         pattern = e.arg.empty? ? '**' : e.arg
         type = e['@type']
         raise ConfigError, "Missing '@type' parameter on <#{e.name}> directive" unless type
@@ -121,7 +122,8 @@ module Fluent
     end
 
     def add_match(type, pattern, conf)
-      log.info :worker0, "adding match#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
+      log_type = conf.for_this_worker? ? :default : :worker0
+      log.info log_type, "adding match#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
 
       output = Plugin.new_output(type)
       output.context_router = @event_router
@@ -142,7 +144,8 @@ module Fluent
     end
 
     def add_filter(type, pattern, conf)
-      log.info :worker0, "adding filter#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
+      log_type = conf.for_this_worker? ? :default : :worker0
+      log.info log_type, "adding filter#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
 
       filter = Plugin.new_filter(type)
       filter.context_router = @event_router

data/lib/fluent/command/fluentd.rb

@@ -158,6 +158,12 @@ op.on('-G', '--gem-path GEM_INSTALL_PATH', "Gemfile install path (default: $(dir
 if Fluent.windows?
   require 'windows/library'
   include Windows::Library
+  
+  opts.merge!(
+    :winsvc_name => 'fluentdwinsvc',
+    :winsvc_display_name => 'Fluentd Windows Service',
+    :winsvc_desc => 'Fluentd is an event collector system.',
+  )
 
   op.on('-x', '--signame INTSIGNAME', "an object name which is used for Windows Service signal (Windows only)") {|s|
     opts[:signame] = s
@@ -174,6 +180,18 @@ if Fluent.windows?
   op.on('--reg-winsvc-fluentdopt OPTION', "specify fluentd option paramters for Windows Service. (Windows only)") {|s|
     opts[:fluentdopt] = s
   }
+  
+  op.on('--winsvc-name NAME', "The Windows Service name to run as (Windows only)") {|s|
+    opts[:winsvc_name] = s
+  }
+  
+  op.on('--winsvc-display-name DISPLAY_NAME', "The Windows Service display name (Windows only)") {|s|
+    opts[:winsvc_display_name] = s
+  }
+  
+  op.on('--winsvc-desc DESC', "The Windows Service description (Windows only)") {|s|
+    opts[:winsvc_desc] = s
+  }
 end
 
 
@@ -229,9 +247,6 @@ end
 early_exit = false
 start_service = false
 if winsvcinstmode = opts[:regwinsvc]
-  FLUENTD_WINSVC_NAME="fluentdwinsvc"
-  FLUENTD_WINSVC_DISPLAYNAME="Fluentd Windows Service"
-  FLUENTD_WINSVC_DESC="Fluentd is an event collector system."
   require 'fileutils'
   require "win32/service"
   require "win32/registry"
@@ -249,27 +264,28 @@ if winsvcinstmode = opts[:regwinsvc]
       start_service = true
     end
 
+    
     Service.create(
-      service_name: FLUENTD_WINSVC_NAME,
+      service_name: opts[:winsvc_name],
       host: nil,
       service_type: Service::WIN32_OWN_PROCESS,
-      description: FLUENTD_WINSVC_DESC,
+      description: opts[:winsvc_desc],
       start_type: start_type,
       error_control: Service::ERROR_NORMAL,
-      binary_path_name: ruby_path+" -C "+binary_path+" winsvc.rb",
+      binary_path_name: "\"#{ruby_path}\" -C \"#{binary_path}\"  winsvc.rb --service-name #{opts[:winsvc_name]}",
       load_order_group: "",
       dependencies: [""],
-      display_name: FLUENTD_WINSVC_DISPLAYNAME
+      display_name: opts[:winsvc_display_name]
     )
   when 'u'
-    if Service.status(FLUENTD_WINSVC_NAME).current_state != 'stopped'
+    if Service.status(opts[:winsvc_name]).current_state != 'stopped'
       begin
-        Service.stop(FLUENTD_WINSVC_NAME)
+        Service.stop(opts[:winsvc_name])
       rescue => ex
         puts "Warning: Failed to stop service: ", ex
       end
     end
-    Service.delete(FLUENTD_WINSVC_NAME)
+    Service.delete(opts[:winsvc_name])
   else
     # none
   end
@@ -277,14 +293,14 @@ if winsvcinstmode = opts[:regwinsvc]
 end
 
 if fluentdopt = opts[:fluentdopt]
-  Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\fluentdwinsvc", Win32::Registry::KEY_ALL_ACCESS) do |reg|
+  Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\#{opts[:winsvc_name]}", Win32::Registry::KEY_ALL_ACCESS) do |reg|
     reg['fluentdopt', Win32::Registry::REG_SZ] = fluentdopt
   end
   early_exit = true
 end
 
 if start_service
-  Service.start(FLUENTD_WINSVC_NAME)
+  Service.start(opts[:winsvc_name])
 end
 
 exit 0 if early_exit

data/lib/fluent/command/plugin_generator.rb

@@ -227,7 +227,7 @@ BANNER
 \tY - yes, overwrite
 \tn - no, do not overwrite
 \ta - all, overwrite this and all others
-\tq - quite, abort
+\tq - quit, abort
 \th - help, show this help
 HELP
       end

data/lib/fluent/compat/detach_process_mixin.rb

@@ -17,9 +17,17 @@
 module Fluent
   module Compat
     module DetachProcessMixin
+      def detach_process
+        log.warn "#{__method__} is not supported in this version. ignored."
+        yield
+      end
     end
 
     module DetachMultiProcessMixin
+      def detach_multi_process
+        log.warn "#{__method__} is not supported in this version. ignored."
+        yield
+      end
     end
   end
 end

data/lib/fluent/compat/input.rb

@@ -44,16 +44,6 @@ module Fluent
       def shutdown
         super
       end
-
-      def detach_process(&block)
-        log.warn "detach_process is not supported in this version. ignored."
-        block.call
-      end
-
-      def detach_multi_process(&block)
-        log.warn "detach_process is not supported in this version. ignored."
-        block.call
-      end
     end
   end
 end

data/lib/fluent/compat/output.rb

@@ -190,16 +190,6 @@ module Fluent
           end
         end
       end
-
-      def detach_process(&block)
-        log.warn "detach_process is not supported in this version. ignored."
-        block.call
-      end
-
-      def detach_multi_process(&block)
-        log.warn "detach_process is not supported in this version. ignored."
-        block.call
-      end
     end
 
     class MultiOutput < Fluent::Plugin::BareOutput

data/lib/fluent/config/element.rb

@@ -35,10 +35,13 @@ module Fluent
 
         # it's global logger, not plugin logger: deprecated message should be global warning, not plugin level.
         @logger = defined?($log) ? $log : nil
+
+        @target_worker_id = nil
       end
 
       attr_accessor :name, :arg, :unused, :v1_config, :corresponding_proxies, :unused_in
       attr_writer :elements
+      attr_reader :target_worker_id
 
       RESERVED_PARAMETERS_COMPAT = {
         '@type' => 'type',
@@ -213,6 +216,25 @@ module Fluent
         v.each_char { |c| result << LiteralParser.unescape_char(c) }
         result
       end
+
+      def set_target_worker_id(worker_id)
+        @target_worker_id = worker_id
+        @elements.each { |e|
+          e.set_target_worker_id(worker_id)
+        }
+      end
+
+      def for_every_workers?
+        @target_worker_id == nil
+      end
+
+      def for_this_worker?
+        @target_worker_id == Fluent::Engine.worker_id
+      end
+
+      def for_another_worker?
+        @target_worker_id != nil && @target_worker_id != Fluent::Engine.worker_id
+      end
     end
   end
 end

data/lib/fluent/engine.rb

@@ -31,6 +31,7 @@ module Fluent
       @root_agent = nil
       @default_loop = nil
       @engine_stopped = false
+      @_worker_id = nil
 
       @log_event_router = nil
       @log_emit_thread = nil
@@ -97,12 +98,21 @@ module Fluent
                     else
                       "section <#{e.name}> is not used in <#{parent_name}>"
                     end
-          $log.warn :worker0, message
+          if e.for_every_workers?
+            $log.warn :worker0, message
+          elsif e.for_this_worker?
+            $log.warn message
+          end
           next
         end
         unless e.name == 'system'
           unless @without_source && e.name == 'source'
-            $log.warn :worker0, "parameter '#{key}' in #{e.to_s.strip} is not used."
+            message = "parameter '#{key}' in #{e.to_s.strip} is not used."
+            if e.for_every_workers?
+              $log.warn :worker0, message
+            elsif e.for_this_worker?
+              $log.warn message
+            end
           end
         end
       }
@@ -128,7 +138,7 @@ module Fluent
 
         unmatched_tags = Fluent::Log.event_tags.select{|t| !@log_event_router.match?(t) }
         unless unmatched_tags.empty?
-          $log.warn :worker0, "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
+          $log.warn "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
         end
       rescue ArgumentError # ArgumentError "#{label_name} label not found"
         # use default event router if <label @FLUENT_LOG> is missing in configuration
@@ -139,7 +149,7 @@ module Fluent
 
           unmatched_tags = Fluent::Log.event_tags.select{|t| !@log_event_router.match?(t) }
           unless unmatched_tags.empty?
-            $log.warn :worker0, "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
+            $log.warn "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
           end
         end
       end
@@ -200,10 +210,6 @@ module Fluent
     end
 
     def run
-      # if ENV doesn't have SERVERENGINE_WORKER_ID, it is a worker under --no-supervisor or in tests
-      # so it's (almost) a single worker, worker_id=0
-      worker_id = (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i
-
       begin
         $log.info "starting fluentd worker", pid: Process.pid, ppid: Process.ppid, worker: worker_id
         start
@@ -251,8 +257,15 @@ module Fluent
       @log_event_queue.push([tag, time, record])
     end
 
-    private
+    def worker_id
+      return @_worker_id if @_worker_id
+      # if ENV doesn't have SERVERENGINE_WORKER_ID, it is a worker under --no-supervisor or in tests
+      # so it's (almost) a single worker, worker_id=0
+      @_worker_id = (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i
+      @_worker_id
+    end
 
+    private
     def start
       @root_agent.start
     end

data/lib/fluent/plugin/base.rb

@@ -51,6 +51,9 @@ module Fluent
       end
 
       def configure(conf)
+        if conf.respond_to?(:for_this_worker?) && conf.for_this_worker?
+          system_config_override(workers: 1)
+        end
         super
         @_state ||= State.new(false, false, false, false, false, false, false, false, false)
         @_state.configure = true

data/lib/fluent/plugin/filter.rb

@@ -79,13 +79,13 @@ module Fluent
         return nil if implmented_methods.include?(:filter_stream)
         case
         when [:filter, :filter_with_time].all? { |e| implmented_methods.include?(e) }
-          raise "BUG: Filter plugins MUST be implemented either `filter` or `filter_with_time`"
+          raise "BUG: Filter plugins MUST implement either `filter` or `filter_with_time`"
         when implmented_methods.include?(:filter)
           false
         when implmented_methods.include?(:filter_with_time)
           true
         else
-          raise NotImplementedError, "BUG: Filter plugins MUST be implmented either `filter` or `filter_with_time`"
+          raise NotImplementedError, "BUG: Filter plugins MUST implement either `filter` or `filter_with_time`"
         end
       end
     end

data/lib/fluent/plugin/in_http.rb

@@ -319,6 +319,8 @@ module Fluent::Plugin
           when /Origin/i
             @origin  = v
           when /X-Forwarded-For/i
+            # For multiple X-Forwarded-For headers. Use first header value.
+            v = v.first if v.is_a?(Array)
             @remote_addr = v.split(",").first
           end
         }

data/lib/fluent/plugin/in_tail.rb

@@ -189,14 +189,17 @@ module Fluent::Plugin
         path = date.strftime(path)
         if path.include?('*')
           paths += Dir.glob(path).select { |p|
-            if File.readable?(p) && !File.directory?(p)
+            is_file = !File.directory?(p)
+            if File.readable?(p) && is_file
               if @limit_recently_modified && File.mtime(p) < (date - @limit_recently_modified)
                 false
               else
                 true
               end
             else
-              log.warn "#{p} unreadable. It is excluded and would be examined next time."
+              if is_file
+                log.warn "#{p} unreadable. It is excluded and would be examined next time."
+              end
               false
             end
           }