logstash-output-file 2.0.2 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 846c9759120f7274317f35fdbd2ef6f15a41ba09
-  data.tar.gz: 5db0140df4d3bec3cb03caf283b39297136746a6
+  metadata.gz: 34e5dcb5b4862273a9cc92f1c4ab73235db7df4a
+  data.tar.gz: 802613ca119351bcb1970d3ec96efa52c18b6a38
 SHA512:
-  metadata.gz: 00c98b731a4fab617426131b03bb89441c710fe99b3744e012dc8c1639d79fd0017a2519e9fcff96c3700ef8137374d236d105068f54460c7767d0f9ef6e710a
-  data.tar.gz: 3f0d377a32cf0f8e55941ab7ff5b67367ea7e656b6de7c9a6858e5eeea89e52d6b6bfd8f60a1c0ef80ef7fbb67ad43241005935cf535f87ced5feb1e14e08c26
+  metadata.gz: ab71908733217054a01c5688cd3a31a6257350440440ba4885d7ac15a8404207b7fd2b19b067790541fd6db40d5e99e3b7cfd9173fc82f3b38c16daa4d4b4f76
+  data.tar.gz: 798c54d4b133f2aed3dba737254dc2807ad3d6f0253a2ae6e06e8b51bee5e7b691415532cab396868b60959a6c92c84ba1eb4424786fccf7a4b978914c338a36

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.1.0
+ - Add create_if_deleted option to create a destination file in case it
+   was deleted by another agent in the machine. In case of being false
+   the system will add the incomming messages to the failure file.
+
 ## 2.0.0
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895

data/README.md

@@ -1,5 +1,8 @@
 # Logstash Plugin
 
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-file-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-file-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/outputs/file.rb

@@ -11,6 +11,8 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
 
   config_name "file"
 
+  attr_reader :failure_path
+
   # The path to the file to write. Event fields can be used here,
   # like `/var/log/logstash/%{host}/%{application}`
   # One may also utilize the path option for date-based log
@@ -42,6 +44,11 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
   # into this file and inside the defined path.
   config :filename_failure, :validate => :string, :default => '_filepath_failures'
 
+  # If the a file is deleted, but an event is comming with the need to be stored
+  # in such a file, the plugin will created a gain this file. Default => true
+  config :create_if_deleted, :validate => :boolean, :default => true
+ 
+
   public
   def register
     require "fileutils" # For mkdir_p
@@ -56,8 +63,10 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
 
     if path_with_field_ref?
       @file_root = extract_file_root
-      @failure_path = File.join(@file_root, @filename_failure)
+    else
+      @file_root = File.dirname(path)
     end
+    @failure_path = File.join(@file_root, @filename_failure)
 
     now = Time.now
     @last_flush_cycle = now
@@ -87,13 +96,14 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
 
   public
   def receive(event)
-    
 
     file_output_path = generate_filepath(event)
 
     if path_with_field_ref? && !inside_file_root?(file_output_path)
       @logger.warn("File: the event tried to write outside the files root, writing the event to the failure file",  :event => event, :filename => @failure_path)
       file_output_path = @failure_path
+    elsif !@create_if_deleted && deleted?(file_output_path)
+      file_output_path = @failure_path
     end
 
     output = format_message(event)
@@ -123,7 +133,6 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
   def write_event(log_path, event)
     @logger.debug("File, writing event to file.", :filename => log_path)
     fd = open(log_path)
-
     # TODO(sissel): Check if we should rotate the file.
 
     fd.write(event)
@@ -198,9 +207,27 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
   end
 
   private
-  def open(path)
-    return @files[path] if @files.include?(path) and not @files[path].nil?
+  def cached?(path)
+     @files.include?(path) && !@files[path].nil?
+  end
 
+  private
+  def deleted?(path)
+    !File.exist?(path)
+  end
+
+  private
+  def open(path)
+    if !deleted?(path) && cached?(path)
+      return @files[path]
+    elsif deleted?(path)
+      if @create_if_deleted
+        @logger.debug("Required path was deleted, creating the file again", :path => path)
+        @files.delete(path)
+      else
+        return @files[path] if cached?(path)
+      end
+    end
     @logger.info("Opening file", :path => path)
 
     dir = File.dirname(path)
@@ -208,13 +235,12 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
       @logger.info("Creating directory", :directory => dir)
       FileUtils.mkdir_p(dir)
     end
-
     # work around a bug opening fifos (bug JRUBY-6280)
     stat = File.stat(path) rescue nil
     if stat && stat.ftype == "fifo" && LogStash::Environment.jruby?
       fd = java.io.FileWriter.new(java.io.File.new(path))
     else
-      fd = File.new(path, "a")
+      fd = File.new(path, "a+")
     end
     if gzip
       fd = Zlib::GzipWriter.new(fd)

data/logstash-output-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-file'
-  s.version         = '2.0.2'
+  s.version         = '2.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This output will write events to files on disk"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/file_spec.rb

@@ -6,6 +6,7 @@ require "logstash/json"
 require "stud/temporary"
 require "tempfile"
 require "uri"
+require "fileutils"
 
 describe LogStash::Outputs::File do
   describe "ship lots of events to a file" do
@@ -106,6 +107,58 @@ describe LogStash::Outputs::File do
   end
 
   describe "receiving events" do
+
+    context "when the output file is deleted" do
+
+      let(:temp_file) { Tempfile.new('logstash-spec-output-file_deleted') }
+
+      let(:config) do
+        { "path" => temp_file.path, "flush_interval" => 0 }
+      end
+
+      it "should recreate the required file if deleted" do
+        output = LogStash::Outputs::File.new(config)
+        output.register
+
+        10.times do |i|
+          event = LogStash::Event.new("event_id" => i)
+          output.receive(event)
+        end
+        FileUtils.rm(temp_file)
+        10.times do |i|
+          event = LogStash::Event.new("event_id" => i+10)
+          output.receive(event)
+        end
+        expect(FileTest.size(temp_file.path)).to be > 0
+      end
+
+      context "when appending to the error log" do
+
+        let(:config) do
+          { "path" => temp_file.path, "flush_interval" => 0, "create_if_deleted" => false }
+        end
+
+        it "should append the events to the filename_failure location" do
+          output = LogStash::Outputs::File.new(config)
+          output.register
+
+          10.times do |i|
+            event = LogStash::Event.new("event_id" => i)
+            output.receive(event)
+          end
+          FileUtils.rm(temp_file)
+          10.times do |i|
+            event = LogStash::Event.new("event_id" => i+10)
+            output.receive(event)
+          end
+          expect(FileTest.exist?(temp_file.path)).to be_falsey
+          expect(FileTest.size(output.failure_path)).to be > 0
+        end
+
+      end
+
+    end
+
     context "when using an interpolated path" do
       context "when trying to write outside the files root directory" do
         let(:bad_event) do

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-file
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2015-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,34 +28,36 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: logstash-input-generator
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-input-generator
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -95,9 +95,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.4.6
 signing_key:
 specification_version: 4
 summary: This output will write events to files on disk
 test_files:
 - spec/outputs/file_spec.rb
+has_rdoc: