logstash-output-file 3.0.2 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 816218ba76195ff85b4c30a0876ec027daa7ae89
-  data.tar.gz: 67ef9d3a53d5b0d94906da6ffa3cc110048edf6e
+  metadata.gz: 7519db3db3ff8bc30ac81db904b4141f518f326b
+  data.tar.gz: 4e0736f6ec2d53be5b3675ac0b2a3015d5a6ac6b
 SHA512:
-  metadata.gz: 16d18bc194e55a7de0e1195db43e4984cc09d872982d9c6df64600e2e02cef4f78b3afa4f0a73692411d14e1ee3b358963e73aa5bd3b3c2f273acd53e146eaee
-  data.tar.gz: 2f51482a17ad47a5889a29c01e87656f3e4ef22534704a70874ddc0a6b55d0fe9afacbf90e854b96bc32757e16f428078a03aeeeea40bbaa30c948f5f96a11d0
+  metadata.gz: 3fdd1d4e5edcb5025969a6298ec457d315386d345ebc961d36c34455ebfa1d0d4b7bc0c3791a83ac5a0dbadc674773fb73a6bb3981b442a42f03858f602d72bf
+  data.tar.gz: 2e357366f22791bd8024845f2ad6ee25e69bd06803cad29957d28c2bdc7899a9ef945bc2c61a123ef79fa4f449865548495ad78b51ef7b17634fc14acf6a0a0f

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.0.0
+  - Make 'message_format' option obsolete
+  - Use new Logsash 2.4/5.0 APIs for working batchwise and with shared concurrency
+
 ## 3.0.2
   - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 

data/lib/logstash/outputs/file.rb

@@ -17,6 +17,8 @@ require "zlib"
 #  }
 # }
 class LogStash::Outputs::File < LogStash::Outputs::Base
+  concurrency :shared
+  
   FIELD_REF = /%\{[^}]+\}/
 
   config_name "file"
@@ -35,13 +37,7 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
   # E.g: `/%{myfield}/`, `/test-%{myfield}/` are not valid paths
   config :path, :validate => :string, :required => true
 
-  # The format to use when writing events to the file. This value
-  # supports any string and can include `%{name}` and other dynamic
-  # strings.
-  #
-  # If this setting is omitted, the full json representation of the
-  # event will be written as a single line.
-  config :message_format, :validate => :string, :deprecated => "You can achieve the same behavior with the 'line' codec"
+  config :message_format, :validate => :string, :obsolete => "You can achieve the same behavior with the 'line' codec"
 
   # Flush interval (in seconds) for flushing writes to log files.
   # 0 will flush on every message.
@@ -76,10 +72,9 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
   def register
     require "fileutils" # For mkdir_p
 
-    workers_not_supported
-
     @files = {}
-
+    @io_mutex = Mutex.new
+    
     @path = File.expand_path(path)
 
     validate_path
@@ -91,6 +86,7 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
     end
     @failure_path = File.join(@file_root, @filename_failure)
 
+
     now = Time.now
     @last_flush_cycle = now
     @last_stale_cleanup_cycle = now
@@ -101,8 +97,6 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
      @codec = LogStash::Plugin.lookup("codec", "line").new
      @codec.format = @message_format
     end
-
-    @codec.on_event(&method(:write_event))
   end # def register
 
   private
@@ -125,20 +119,37 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
   end
 
   public
-  def receive(event)
-    @codec.encode(event)
-    close_stale_files
+  def multi_receive_encoded(events_and_encoded)
+    encoded_by_path = Hash.new {|h,k| h[k] = []}
+
+    events_and_encoded.each do |event,encoded|
+      file_output_path = event_path(event)
+      encoded_by_path[file_output_path] << encoded
+    end
+
+    @io_mutex.synchronize do
+      encoded_by_path.each do |path,chunks|
+        fd = open(path)
+        chunks.each {|chunk| fd.write(chunk) }
+        fd.flush
+      end
+      
+      close_stale_files
+    end   
   end # def receive
 
   public
   def close
-    @logger.debug("Close: closing files")
-    @files.each do |path, fd|
-      begin
-        fd.close
-        @logger.debug("Closed file #{path}", :fd => fd)
-      rescue Exception => e
-        @logger.error("Exception while flushing and closing files.", :exception => e)
+    @io_mutex.synchronize do
+      @logger.debug("Close: closing files")
+      
+      @files.each do |path, fd|
+        begin
+          fd.close
+          @logger.debug("Closed file #{path}", :fd => fd)
+        rescue Exception => e
+          @logger.error("Exception while flushing and closing files.", :exception => e)
+        end
       end
     end
   end
@@ -150,7 +161,7 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
   end
 
   private
-  def write_event(event, data)
+  def event_path(event)
     file_output_path = generate_filepath(event)
     if path_with_field_ref? && !inside_file_root?(file_output_path)
       @logger.warn("File: the event tried to write outside the files root, writing the event to the failure file",  :event => event, :filename => @failure_path)
@@ -159,10 +170,8 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
       file_output_path = @failure_path
     end
     @logger.debug("File, writing event to file.", :filename => file_output_path)
-    fd = open(file_output_path)
-    # TODO(sissel): Check if we should rotate the file.
-    fd.write(data)
-    flush(fd)
+    
+    file_output_path    
   end
 
   private
@@ -195,10 +204,12 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
   def flush_pending_files
     return unless Time.now - @last_flush_cycle >= flush_interval
     @logger.debug("Starting flush cycle")
+
     @files.each do |path, fd|
       @logger.debug("Flushing file", :path => path, :fd => fd)
       fd.flush
     end
+    
     @last_flush_cycle = Time.now
   end
 
@@ -207,6 +218,7 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
   def close_stale_files
     now = Time.now
     return unless now - @last_stale_cleanup_cycle >= @stale_cleanup_interval
+
     @logger.info("Starting stale files cleanup cycle", :files => @files)
     inactive_files = @files.select { |path, fd| not fd.active }
     @logger.debug("%d stale files found" % inactive_files.count, :inactive_files => inactive_files)
@@ -222,7 +234,7 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
 
   private
   def cached?(path)
-     @files.include?(path) && !@files[path].nil?
+    @files.include?(path) && !@files[path].nil?
   end
 
   private
@@ -234,7 +246,9 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
   def open(path)
     if !deleted?(path) && cached?(path)
       return @files[path]
-    elsif deleted?(path)
+    end
+
+    if deleted?(path)
       if @create_if_deleted
         @logger.debug("Required path was deleted, creating the file again", :path => path)
         @files.delete(path)
@@ -242,8 +256,9 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
         return @files[path] if cached?(path)
       end
     end
-    @logger.info("Opening file", :path => path)
 
+    @logger.info("Opening file", :path => path)
+    
     dir = File.dirname(path)
     if !Dir.exist?(dir)
       @logger.info("Creating directory", :directory => dir)
@@ -253,6 +268,7 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
         FileUtils.mkdir_p(dir)
       end
     end
+    
     # work around a bug opening fifos (bug JRUBY-6280)
     stat = File.stat(path) rescue nil
     if stat && stat.ftype == "fifo" && LogStash::Environment.jruby?
@@ -288,6 +304,7 @@ class IOWriter
   end
   def method_missing(method_name, *args, &block)
     if @io.respond_to?(method_name)
+
       @io.send(method_name, *args, &block)
     else
       super

data/logstash-output-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-file'
-  s.version         = '3.0.2'
+  s.version         = '4.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This output will write events to files on disk"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 2.0.0", "< 2.99"
   s.add_runtime_dependency 'logstash-codec-json_lines'
   s.add_runtime_dependency 'logstash-codec-line'
 

data/spec/outputs/file_spec.rb

@@ -47,7 +47,7 @@ describe LogStash::Outputs::File do
 
   describe "ship lots of events to a file gzipped" do
     Stud::Temporary.file('logstash-spec-output-file') do |tmp_file|
-      event_count = 10000 + rand(500)
+      event_count = 100000 + rand(500)
 
       config <<-CONFIG
         input {
@@ -125,13 +125,14 @@ describe LogStash::Outputs::File do
 
         10.times do |i|
           event = LogStash::Event.new("event_id" => i)
-          output.receive(event)
+          output.multi_receive([event])
         end
         FileUtils.rm(temp_file)
         10.times do |i|
           event = LogStash::Event.new("event_id" => i+10)
-          output.receive(event)
+          output.multi_receive([event])
         end
+        
         expect(FileTest.size(temp_file.path)).to be > 0
       end
 
@@ -147,12 +148,12 @@ describe LogStash::Outputs::File do
 
           10.times do |i|
             event = LogStash::Event.new("event_id" => i)
-            output.receive(event)
+            output.multi_receive([event])
           end
           FileUtils.rm(temp_file)
           10.times do |i|
             event = LogStash::Event.new("event_id" => i+10)
-            output.receive(event)
+            output.multi_receive([event])
           end
           expect(FileTest.exist?(temp_file.path)).to be_falsey
           expect(FileTest.size(output.failure_path)).to be > 0
@@ -184,7 +185,7 @@ describe LogStash::Outputs::File do
 
             output = LogStash::Outputs::File.new(config)
             output.register
-            output.receive(bad_event)
+            output.multi_receive([bad_event])
 
             error_file = File.join(path, config["filename_failure"])
 
@@ -202,10 +203,10 @@ describe LogStash::Outputs::File do
             output.register
 
             bad_event.set('error', encoded_once)
-            output.receive(bad_event)
+            output.multi_receive([bad_event])
 
             bad_event.set('error', encoded_twice)
-            output.receive(bad_event)
+            output.multi_receive([bad_event])
 
             expect(Dir.glob(File.join(path, "*")).size).to eq(2)
             output.close
@@ -218,7 +219,7 @@ describe LogStash::Outputs::File do
             output.register
 
             bad_event.set('error', '../..//test')
-            output.receive(bad_event)
+            output.multi_receive([bad_event])
 
             expect(Dir.glob(File.join(path, "*")).size).to eq(1)
             output.close
@@ -235,7 +236,7 @@ describe LogStash::Outputs::File do
             config = { "path" => "#{path}/%{error}" }
             output = LogStash::Outputs::File.new(config)
             output.register
-            output.receive(good_event)
+            output.multi_receive([good_event])
 
             good_file = File.join(path, good_event.get('error'))
             expect(File.exist?(good_file)).to eq(true)
@@ -254,7 +255,7 @@ describe LogStash::Outputs::File do
             config = { "path" => dynamic_path }
             output = LogStash::Outputs::File.new(config)
             output.register
-            output.receive(good_event)
+            output.multi_receive([good_event])
 
             expect(File.exist?(expected_path)).to eq(true)
             output.close
@@ -276,7 +277,7 @@ describe LogStash::Outputs::File do
 
             output = LogStash::Outputs::File.new(config)
             output.register
-            output.receive(good_event)
+            output.multi_receive([good_event])
 
             expect(File.exist?(expected_path)).to eq(true)
             output.close
@@ -291,7 +292,7 @@ describe LogStash::Outputs::File do
             config = { "path" => "#{path}/%{error}" }
             output = LogStash::Outputs::File.new(config)
             output.register
-            output.receive(good_event)
+            output.multi_receive([good_event])
 
             good_file = File.join(path, good_event.get('error'))
             expect(File.exist?(good_file)).to eq(true)
@@ -310,7 +311,7 @@ describe LogStash::Outputs::File do
             config = { "path" => "#{path}/output.txt" }
             output = LogStash::Outputs::File.new(config)
             output.register
-            output.receive(good_event)
+            output.multi_receive([good_event])
             good_file = File.join(path, 'output.txt')
             expect(File.exist?(good_file)).to eq(true)
             output.close #teardown first to allow reading the file
@@ -328,30 +329,9 @@ describe LogStash::Outputs::File do
 
           Stud::Temporary.directory do |path|
             config = { "path" => "#{path}/output.txt" }
-            output = LogStash::Outputs::File.new(config)
-            output.codec = LogStash::Codecs::Line.new({ "format" => "Custom format: %{message}"})
-            output.register
-            output.receive(good_event)
-            good_file = File.join(path, 'output.txt')
-            expect(File.exist?(good_file)).to eq(true)
-            output.close #teardown first to allow reading the file
-            File.open(good_file) {|f|
-              line = f.readline
-              expect(line).to eq("Custom format: hello world\n")
-            }
-          end
-        end
-      end
-      context "when using deprecated message_format config" do
-        it 'falls back to line codec' do
-          good_event = LogStash::Event.new
-          good_event.set('message', 'hello world')
-
-          Stud::Temporary.directory do |path|
-            config = { "path" => "#{path}/output.txt", "message_format" => "Custom format: %{message}" }
-            output = LogStash::Outputs::File.new(config)
+            output = LogStash::Outputs::File.new(config.merge("codec" => LogStash::Codecs::Line.new({ "format" => "Custom format: %{message}"})))
             output.register
-            output.receive(good_event)
+            output.multi_receive([good_event])
             good_file = File.join(path, 'output.txt')
             expect(File.exist?(good_file)).to eq(true)
             output.close #teardown first to allow reading the file
@@ -375,7 +355,7 @@ describe LogStash::Outputs::File do
             }
             output = LogStash::Outputs::File.new(config)
             output.register
-            output.receive(good_event)
+            output.multi_receive([good_event])
             good_file = File.join(path, 'is/nested/output.txt')
             expect(File.exist?(good_file)).to eq(true)
             expect(File.stat(good_file).mode.to_s(8)[-3..-1]).to eq('610')

metadata

@@ -1,22 +1,22 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-file
 version: !ruby/object:Gem::Version
-  version: 3.0.2
+  version: 4.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2016-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.60'
-    - - "<="
+        version: 2.0.0
+    - - "<"
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
@@ -26,8 +26,8 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.60'
-    - - "<="
+        version: 2.0.0
+    - - "<"
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
@@ -123,7 +123,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: This output will write events to files on disk