fluentd 0.14.22.rc1 → 0.14.22.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cabaaf075c80b44f6b8f77efb057ad809212f871
-  data.tar.gz: 7092d208bb92fe4992764dbe418b117976a7ba66
+  metadata.gz: 2e01e6085ba9a64e0d45542f56d76ea2f1cbd0ec
+  data.tar.gz: ecc01811bfe9bd014e42b2f338475428aecec35c
 SHA512:
-  metadata.gz: 235273ae1d58e4e3c3be98642805cb9d62ac3df2c75cc94622ee78bf62b65e5927aec1ad9ba2043f4c06b65abed6a490340243fb1b82d42e6e0e8e9bd8d4a500
-  data.tar.gz: dcab5670899cb22e7914cffcb2a31f590d054f31ec44b500720cbbc02e59c4f22a9da02b2078035dc5dbf7eea625c34f60f8002eb9c83271c60f0b02883a0a9c
+  metadata.gz: 9f1895eea93d665cd36160f1290c9d6898b7f065477e60a873792ae17769412bf4ede53455bc10fc9338c71cf1b8dc8a18f1638237b3195b21a7448beec06243
+  data.tar.gz: c19abd745f755d4fc27b8702b7ccd8f0e312ab6fd13c53bdd5b6bf577c19a283d5979c3744de4fffff4c88947d39d7cbc0376edf314de511c844c543af718deb

data/lib/fluent/plugin/base.rb

@@ -156,6 +156,18 @@ module Fluent
         @_state.terminate
       end
 
+      def called_in_test?
+        caller_locations.each do |location|
+          # Thread::Backtrace::Location#path returns base filename or absolute path.
+          # #absolute_path returns absolute_path always.
+          # https://bugs.ruby-lang.org/issues/12159
+          if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
+            return true
+          end
+        end
+        false
+      end
+
       def inspect
         # Plugin instances are sometimes too big to dump because it may have too many thins (buffer,storage, ...)
         # Original commit comment says that:

data/lib/fluent/plugin/buf_file.rb

@@ -66,7 +66,7 @@ module Fluent
         end
 
         type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
-        if @@buffer_paths.has_key?(@path) && !buffer_path_for_test?
+        if @@buffer_paths.has_key?(@path) && !called_in_test?
           type_using_this_path = @@buffer_paths[@path]
           raise ConfigError, "Other '#{type_using_this_path}' plugin already use same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
         end
@@ -114,18 +114,6 @@ module Fluent
         @multi_workers_available
       end
 
-      def buffer_path_for_test?
-        caller_locations.each do |location|
-          # Thread::Backtrace::Location#path returns base filename or absolute path.
-          # #absolute_path returns absolute_path always.
-          # https://bugs.ruby-lang.org/issues/12159
-          if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
-            return true
-          end
-        end
-        false
-      end
-
       def start
         FileUtils.mkdir_p File.dirname(@path), mode: @dir_permission
 
@@ -170,10 +158,14 @@ module Fluent
       def generate_chunk(metadata)
         # FileChunk generates real path with unique_id
         if @file_permission
-          Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: @file_permission, compress: @compress)
+          chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: @file_permission, compress: @compress)
         else
-          Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, compress: @compress)
+          chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, compress: @compress)
         end
+
+        log.debug "Created new chunk", chunk_id: dump_unique_id_hex(chunk.unique_id), metadata: metadata
+
+        return chunk
       end
     end
   end

data/lib/fluent/plugin/buffer.rb

@@ -313,22 +313,24 @@ module Fluent
 
       def enqueue_chunk(metadata)
         log.trace "enqueueing chunk", instance: self.object_id, metadata: metadata
-        synchronize do
-          chunk = @stage.delete(metadata)
-          return nil unless chunk
+        chunk = synchronize do
+          @stage.delete(metadata)
+        end
+        return nil unless chunk
 
-          chunk.synchronize do
+        chunk.synchronize do
+          synchronize do
             if chunk.empty?
               chunk.close
             else
               @queue << chunk
               @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
-              chunk.enqueued! if chunk.respond_to?(:enqueued!)
+              chunk.enqueued!
             end
+            bytesize = chunk.bytesize
+            @stage_size -= bytesize
+            @queue_size += bytesize
           end
-          bytesize = chunk.bytesize
-          @stage_size -= bytesize
-          @queue_size += bytesize
         end
         nil
       end
@@ -340,7 +342,7 @@ module Fluent
             metadata = chunk.metadata
             @queue << chunk
             @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
-            chunk.enqueued! if chunk.respond_to?(:enqueued!)
+            chunk.enqueued!
           end
           @queue_size += chunk.bytesize
         end
@@ -348,17 +350,18 @@ module Fluent
 
       def enqueue_all
         log.trace "enqueueing all chunks in buffer", instance: self.object_id
-        synchronize do
-          if block_given?
-            @stage.keys.each do |metadata|
-              chunk = @stage[metadata]
-              v = yield metadata, chunk
-              enqueue_chunk(metadata) if v
-            end
-          else
-            @stage.keys.each do |metadata|
-              enqueue_chunk(metadata)
-            end
+        if block_given?
+          synchronize{ @stage.keys }.each do |metadata|
+            # NOTE: The following line might cause data race depending on Ruby implementations except CRuby
+            # cf. https://github.com/fluent/fluentd/pull/1721#discussion_r146170251
+            chunk = @stage[metadata]
+            next unless chunk
+            v = yield metadata, chunk
+            enqueue_chunk(metadata) if v
+          end
+        else
+          synchronize{ @stage.keys }.each do |metadata|
+            enqueue_chunk(metadata)
           end
         end
       end

data/lib/fluent/plugin/in_tail.rb

@@ -87,6 +87,8 @@ module Fluent::Plugin
 
     attr_reader :paths
 
+    @@pos_file_paths = {}
+
     def configure(conf)
       compat_parameters_convert(conf, :parser)
       parser_config = conf.elements('parse').first
@@ -109,7 +111,13 @@ module Fluent::Plugin
       end
 
       # TODO: Use plugin_root_dir and storage plugin to store positions if available
-      unless @pos_file
+      if @pos_file
+        if @@pos_file_paths.has_key?(@pos_file) && !called_in_test?
+          plugin_id_using_this_path = @@pos_file_paths[@pos_file]
+          raise Fluent::ConfigError, "Other 'in_tail' plugin already use same pos_file path: plugin_id = #{plugin_id_using_this_path}, pos_file path = #{@pos_file}"
+        end
+        @@pos_file_paths[@pos_file] = self.plugin_id
+      else
         $log.warn "'pos_file PATH' parameter is not set to a 'tail' source."
         $log.warn "this parameter is highly recommended to save the position to resume tailing."
       end

data/lib/fluent/supervisor.rb

@@ -513,7 +513,7 @@ module Fluent
       install_main_process_signal_handlers
 
       # This is the only log messsage for @standalone_worker
-      $log.info "starting fluentd-#{Fluent::VERSION} without supervision", pid: Process.pid if @standalone_worker
+      $log.info "starting fluentd-#{Fluent::VERSION} without supervision", pid: Process.pid, ruby: RUBY_VERSION if @standalone_worker
 
       main_process do
         create_socket_manager if @standalone_worker
@@ -535,7 +535,7 @@ module Fluent
     end
 
     def dry_run_cmd
-      $log.info "starting fluentd-#{Fluent::VERSION} as dry run mode"
+      $log.info "starting fluentd-#{Fluent::VERSION} as dry run mode", ruby: RUBY_VERSION
       @system_config.suppress_config_dump = true
       dry_run
       exit 0
@@ -571,7 +571,7 @@ module Fluent
       dry_run
 
       Process.setproctitle("supervisor:#{@process_name}") if @process_name
-      $log.info "starting fluentd-#{Fluent::VERSION}", pid: Process.pid
+      $log.info "starting fluentd-#{Fluent::VERSION}", pid: Process.pid, ruby: RUBY_VERSION
 
       rubyopt = ENV["RUBYOPT"]
       fluentd_spawn_cmd = [ServerEngine.ruby_bin_path, "-Eascii-8bit:ascii-8bit"]

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.22.rc1'
+  VERSION = '0.14.22.rc2'
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.14.22.rc1
+  version: 0.14.22.rc2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-13 00:00:00.000000000 Z
+date: 2017-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -740,7 +740,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.12
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector