logstash-output-file_closeable 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: eb6ffd81954405585db0c421aca83f9ef69f5818
+  data.tar.gz: 8b6043f66938682c71f0430a9d49c7273b20ffb2
+SHA512:
+  metadata.gz: 04227da02a03ca3c9df7c67d13f937a5b426ce3ead1182292d67798a1d410a2965a54f5bf6a6793ae8b740597f26a5ee81c660ed4654f941ab069c53008dcf00
+  data.tar.gz: 536851937ba38a919215d58fe687a5a180be615b38a40746743165b36d4bdd95879fa35cc3f0f3d52580faa750fca3617e85dd0e61276f1f3bdd657bfd6ba873

data/.gitignore

@@ -0,0 +1,3 @@
+Gemfile.lock
+.ruby-version
+*.gem

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -0,0 +1,7 @@
+@files=[]
+
+task :default do
+  system("rake -T")
+end
+
+require "logstash/devutils/rake"

data/lib/logstash/outputs/file_closeable.rb

@@ -0,0 +1,195 @@
+require "logstash/namespace"
+require "logstash/outputs/base"
+require "zlib"
+
+# File output.
+#
+# Write events to files on disk. You can use fields from the
+# event as parts of the filename.
+class LogStash::Outputs::FileCloseable < LogStash::Outputs::Base
+
+    if RUBY_ENGINE == "jruby"
+        JavaException = java.lang.Exception
+    end
+
+    config_name "file2"
+
+    # The format to use when writing events to the file. This value
+    # supports any string and can include %{name} and other dynamic
+    # strings.
+    #
+    # If this setting is omitted, the full json representation of the
+    # event will be written as a single line.
+    config :message_format, :validate => :string
+
+    # Flush interval for flushing writes to log files. 0 will flush on every meesage
+    config :flush_interval, :validate => :number, :default => 2
+
+    # Gzip output stream
+    config :gzip, :validate => :boolean, :default => false
+
+    # The path to the file to write. Event fields can be used here,
+    # like "/var/log/logstash/%{host}/%{application}"
+    # One may also utilize the path option for date-based log
+    # rotation via the joda time format. This will use the event
+    # timestamp.
+    # E.g.: path => "./test-%{+YYYY-MM-dd}.txt" to create
+    # ./test-2013-05-29.txt
+    config :path, :validate => :string
+
+    public
+    def register
+        require "fileutils" # For mkdir_p
+
+        @files = {}
+        now = Time.now
+        @last_flush_cycle = now
+        @last_stale_cleanup_cycle = now
+        flush_interval = @flush_interval.to_i
+        @stale_cleanup_interval = 10
+
+        @codec.on_event do |event|
+            if @path
+                path = event.sprintf(@path)
+            else
+                path = event["path"]
+            end
+
+            fd = open(path)
+
+            if event.is_a? LogStash::Event and @message_format
+                output = event.sprintf(@message_format)
+            else
+                output = event["message"]
+            end
+
+            @logger.debug("Writing output to file", :output => output, :path => path) if @logger.debug?
+
+            fd.write(output)
+            fd.write("\n")
+
+            flush(fd)
+            close_stale_files(false)
+        end
+    end # def register
+
+    public
+    def receive(event)
+        return unless output?(event)
+
+        @codec.encode(event)
+
+        if event.include? "tags" and event["tags"].include?("eof")
+
+            @files.each do |path, fd|
+                if(File.basename(path) == File.basename(event['path']))
+                    fd.active = false
+                end
+            end
+
+            close_stale_files(true) #means eof
+            return
+        end
+    end # def receive
+
+    def teardown
+        @logger.debug("Teardown: closing files") if @logger.debug?
+        @files.each do |path, fd|
+            begin
+                fd.close
+                @logger.debug("Closed file #{path}", :fd => fd) if @logger.debug?
+            rescue Exception => e
+                @logger.error("Excpetion while flushing and closing files.", :exception => e) if @logger.debug?
+            end
+        end
+        finished
+    end
+
+    private
+    def flush(fd)
+        if flush_interval > 0
+            flush_pending_files
+        else
+            fd.flush
+        end
+    end
+
+    # every flush_interval seconds or so (triggered by events, but if there are no events there's no point flushing files anyway)
+    def flush_pending_files
+        return unless Time.now - @last_flush_cycle >= flush_interval
+
+        @logger.debug("Starting flush cycle") if @logger.debug?
+        @files.each do |path, fd|
+            @logger.debug("Flushing file", :path => path, :fd => fd) if @logger.debug?
+            fd.flush
+        end
+        @last_flush_cycle = Time.now
+    end
+
+    # every 10 seconds or so (triggered by events, but if there are no events there's no point closing files anyway)
+    def close_stale_files(eof)
+        now = Time.now
+        return if now - @last_stale_cleanup_cycle < @stale_cleanup_interval and not eof
+
+        @logger.info("Starting stale files cleanup cycle", :files => @files)
+        inactive_files = @files.select { |path, fd| not fd.active }
+        @logger.debug("%d stale files found" % inactive_files.count, :inactive_files => inactive_files) if @logger.debug?
+        inactive_files.each do |path, fd|
+          @logger.info("Closing file %s" % path)
+          fd.close
+          @files.delete(path)
+        end
+        # mark all files as inactive, a call to write will mark them as active again
+        @files.each { |path, fd| fd.active = false }
+        @last_stale_cleanup_cycle = now
+    end
+
+    def open(path)
+        return @files[path] if @files.include?(path) and not @files[path].nil?
+
+        @logger.info("Opening file", :path => path)
+
+        dir = File.dirname(path)
+        if !Dir.exists?(dir)
+            @logger.info("Creating directory", :directory => dir)
+            FileUtils.mkdir_p(dir)
+        end
+
+        # work around a bug opening fifos (bug JRUBY-6280)
+        stat = File.stat(path) rescue nil
+        if stat and stat.ftype == "fifo" and RUBY_PLATFORM == "java"
+            fd = java.io.FileWriter.new(java.io.File.new(path))
+        else
+            fd = File.new(path, "a")
+        end
+        if gzip
+            fd = Zlib::GzipWriter.new(fd)
+        end
+        @files[path] = IOWriter.new(fd)
+    end
+  end # class LogStash::Outputs::TriggeredFolder
+
+  # wrapper class
+  class IOWriter
+      def initialize(io)
+          @io = io
+      end
+      def write(*args)
+          @io.write(*args)
+          @active = true
+      end
+      def flush
+          @io.flush
+          if @io.class == Zlib::GzipWriter
+            @io.to_io.flush
+          end
+      end
+      def method_missing(method_name, *args, &block)
+          if @io.respond_to?(method_name)
+              @io.send(method_name, *args, &block)
+          else
+              super
+          end
+      end
+      attr_accessor :active
+  end

data/logstash-output-file_closeable.gemspec

@@ -0,0 +1,27 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-output-file_closeable'
+  s.version         = '0.1.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "This output will write events to files on disk with additional closing file of finished reading"
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Signify"]
+  s.email           = 'dietmar@signifydata.com'
+  s.homepage        = "http://www.signifydata.com"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core', '>= 1.4.0', '< 2.0.0'
+
+  s.add_development_dependency 'logstash-devutils'
+end
+

data/spec/outputs/file_closeable_spec.rb

@@ -0,0 +1,5 @@
+require "logstash/devutils/rspec/spec_helper"
+require 'logstash/inputs/file_closeable'
+
+describe LogStash::Inputs::FileCloseable do
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-file_closeable
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Signify
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-08-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: dietmar@signifydata.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- Rakefile
+- lib/logstash/outputs/file_closeable.rb
+- logstash-output-file_closeable.gemspec
+- spec/outputs/file_closeable_spec.rb
+homepage: http://www.signifydata.com
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: This output will write events to files on disk with additional closing file of finished reading
+test_files:
+- spec/outputs/file_closeable_spec.rb