logstash-output-hdfs 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d90d1ec8104476f6daf5848f37a79db8bfd2c6e0
+  data.tar.gz: 14642e70c7219a7852433e65c51af480164a52c6
+SHA512:
+  metadata.gz: 3d65c7f8919758f88022b25956f71f79c6f51cd0bfc539f7f00cf54a98c1625fd5150af567facd910b9043a0297a88acdde0cb9c350ab4327b4d31debe15e0fc
+  data.tar.gz: 2197df4e76cd7ced8f5d57f41989292c85465baedf8fd6dc46da6082a4c0a3877725f78f86e6a2e56c85576ca1c75060250be336438888b3dcb0cbfb1d7e57d7

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/LICENSE.txt

@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

data/README.md

@@ -0,0 +1,75 @@
+# Logstash HDFS plugin
+
+An HDFS plugin for [Logstash](http://logstash.net). This plugin is provided as an external plugin (see Usage below) and is not part of the Logstash project.
+
+# Usage
+
+## Logstash 1.4.x
+
+Run logstash with the `--pluginpath` (`-p`) command line argument to let logstash know where the plugin is. Also, you need to let Java know where your Hadoop JARs are, so set the `CLASSPATH` variable correctly.
+On Logstash 1.4.x use the following command (ajusting paths as neccessary of course):
+
+    LD_LIBRARY_PATH="/usr/lib/hadoop/lib/native" GEM_HOME=./logstash-1.4.2/vendor/bundle/jruby/1.9 CLASSPATH=$(find ./logstash-1.4.2/vendor/jar -type f -name '*.jar'|tr '\n' ':'):$(find /usr/lib/hadoop-hdfs -type f -name '*.jar' | tr '\n' ':'):$(find /usr/lib/hadoop -type f -name '*.jar' | tr '\n' ':'):/etc/hadoop/conf java org.jruby.Main -I./logstash-1.4.2/lib ./logstash-1.4.2/lib/logstash/runner.rb agent -f logstash.conf -p ./logstash-hdfs/lib
+
+Note that logstash is not executed with `java -jar` because executable jars ignore external classpath. Instead we put the logstash jar on the class path and call the runner class.
+Important: the Hadoop configuration dir containing `hdfs-site.xml` must be on the classpath.
+
+## Logstash 1.5.x
+
+Logstash 1.5.x supports distribution of plugins as rubygems which makes life a log easier. To install the plugin from the version on rubygems:
+
+    $LOGSTASH_DIR/bin/plugin install logstash-output-hdfs
+
+Or from source (after checking out the source, run in checkout directory):
+
+    $LOGSTASH_DIR/bin/plugin build logstash-output-hdfs.gemspec
+    
+Then run logstash with the following command:
+
+    LD_LIBRARY_PATH="$HADOOP_DIR/lib/native" CLASSPATH=$(find $HADOOP_DIR/share/hadoop/common/lib/ -name '*.jar' | tr '\n' ':'):$HADOOP_DIR/share/hadoop/hdfs/hadoop-hdfs-2.4.0.jar:$HADOOP_DIR/share/hadoop/common/hadoop-common-2.4.0.jar:$HADOOP_DIR/conf $LOGSTASH_DIR/bin/logstash agent -f logstash.conf
+
+Hadoop paths may need adjustments depending on the distribution and version you are using. The important thing is to have `hadoop-hdfs`, `hadoop-common` and all the jar files in `common/lib` on the classpath.
+
+The following comman line will work on most distributions (but will take a little longer to load since it loads many unnecessary jars):
+
+    LD_LIBRARY_PATH="/usr/lib/hadoop/lib/native" CLASSPATH=$(find /usr/lib/hadoop-hdfs -type f -name '*.jar' | tr '\n' ':'):$(find /usr/lib/hadoop -type f -name '*.jar' | tr '\n' ':'):/etc/hadoop/conf $LOGSTASH_DIR/bin/logstash agent -f logstash.conf
+ 
+
+# HDFS Configuration
+
+By default, the plugin will load Hadoop's configuration from the classpath.  However, a logstash configuration option named 'hadoop_config_resources' has
+been added that will allow the user to pass in multiple configuration locations to override this default configuration.
+
+    output {
+            hdfs {
+                path => "/path/to/output_file.log"
+                hadoop_config_resources => ['path/to/configuration/on/classpath/hdfs-site.xml', 'path/to/configuration/on/classpath/core-site.xml']
+            }
+        }
+
+
+# HDFS Append and rewriting files
+
+Please note, HDFS versions prior to 2.x do not properly support append. See [HADOOP-8230](https://issues.apache.org/jira/browse/HADOOP-8230) for reference.
+To enable append on HDFS, set _dfs.support.append_ in <tt>hdfs-site.conf</tt> (2.x) or _dfs.support.broken.append_ on 1.x, and use the *enable_append* config option:
+
+    output {
+        hdfs {
+            path => "/path/to/output_file.log"
+            enable_append => true
+        }
+    }
+
+If append is not supported and the file already exists, the plugin will cowardly refuse to reopen the file for writing unless *enable_reopen* is set to true.
+This is probably a very bad idea, you have been warned!
+
+# HFDS Flush
+
+Flush and sync don't actually work as promised on HDFS (see [HDFS-536](https://issues.apache.org/jira/browse/HDFS-536)).
+In Hadoop 2.x, `hflush` provides flush-like functionality and the plugin will use `hflush` if it is available.
+Nevertheless, flushing code has been left in the plugin in case `flush` and `sync` will work on some HDFS implementation.
+
+# License
+
+The plugin is released under the LGPL v3.
+

data/Rakefile

@@ -0,0 +1,7 @@
+@files=[]
+
+task :default do
+  system("rake -T")
+end
+
+require "logstash/devutils/rake"

data/lib/logstash/outputs/hdfs.rb

@@ -0,0 +1,213 @@
+require "logstash/namespace"
+require "logstash/outputs/base"
+
+# HDFS output.
+#
+# Write events to files to HDFS. You can use fields from the
+# event as parts of the filename.
+class LogStash::Outputs::HDFS < LogStash::Outputs::Base
+
+  config_name "hdfs"
+  milestone 1
+
+  # The path to the file to write. Event fields can be used here, 
+  # like "/var/log/logstash/%{@source_host}/%{application}"
+  config :path, :validate => :string, :required => true
+
+  # The format to use when writing events to the file. This value
+  # supports any string and can include %{name} and other dynamic
+  # strings.
+  #
+  # If this setting is omitted, the full json representation of the
+  # event will be written as a single line.
+  config :message_format, :validate => :string
+
+  # Flush interval for flushing writes to log files. 0 will flush on every message
+  # Flush doesn't actually work on most Hadoop 1.x versions. if you really care about flush, use 2.x 
+  config :flush_interval, :validate => :number, :default => 60
+
+  # Enable the use of append. This only works with Hadoop 2.x dfs.support.append or 1.x with dfs.support.broken.append
+  config :enable_append, :validate => :boolean, :default => false
+
+  # Enable re-opening files. This is a really a bad idea because HDFS will truncate files. Only use if you know what you're doing
+  config :enable_reopen, :validate => :boolean, :default => false
+
+  # The classpath resource locations of the hadoop configuration
+  config :hadoop_config_resources, :validate => :array
+
+  public
+  def register
+    require "java"
+    java_import "org.apache.hadoop.fs.Path"
+    java_import "org.apache.hadoop.fs.FileSystem"
+    java_import "org.apache.hadoop.conf.Configuration"
+
+    @files = {}
+    now = Time.now
+    @last_flush_cycle = now
+    @last_stale_cleanup_cycle = now
+    flush_interval = @flush_interval.to_i
+    @stale_cleanup_interval = 10
+    conf = Configuration.new
+
+    if @hadoop_config_resources
+      @hadoop_config_resources.each { |resource|
+        conf.addResource(resource)
+      }
+    end
+
+    @logger.info "Using Hadoop configuration: #{conf.get("fs.defaultFS")}"
+    @hdfs = FileSystem.get(conf)
+  end # def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+    out = get_output_stream(event.sprintf(@path))
+
+    if @message_format
+      output = event.sprintf(@message_format)
+    else
+      output = event.to_json
+    end
+    output += "\n" unless output.end_with? "\n"
+
+    out.write(output)
+
+    flush(out)
+    close_stale_files
+  end # def receive
+
+  def teardown
+    @logger.debug("Teardown: closing files")
+    @files.each do |path, fd|
+      begin
+        fd.close
+        @logger.debug("Closed file #{path}", :fd => fd)
+      rescue Exception => e
+        @logger.error("Excpetion while flushing and closing files.", :exception => e)
+      end
+    end
+    finished
+  end
+
+  private
+  def get_output_stream(path_string)
+    return @files[path_string] if @files.has_key?(path_string)
+    path = Path.new(path_string)
+    if @hdfs.exists(path)
+      if enable_append
+        begin
+          dfs_data_output_stream = @hdfs.append(path)
+        rescue java.io.IOException => e
+          logger.error("Error opening path for append, trying to recover lease", :exception => e)
+          recover_lease(path)
+          retry
+        end
+      elsif enable_reopen
+        logger.warn "Overwritting HDFS file", :path => path_string
+        dfs_data_output_stream = @hdfs.create(path, true)
+      else
+        raise IOError, "Cowardly refusing to open pre existing file (#{path_string}) because HDFS will truncate the file!"
+      end
+    else
+      dfs_data_output_stream = @hdfs.create(path)
+    end
+    @files[path_string] = DFSOutputStreamWrapper.new(dfs_data_output_stream)
+  end
+
+  def flush(fd)
+    if flush_interval > 0
+      flush_pending_files
+    else
+      fd.flush
+    end
+  end
+
+  # every flush_interval seconds or so (triggered by events, but if there are no events there's no point flushing files anyway)
+  def flush_pending_files
+    return unless Time.now - @last_flush_cycle >= flush_interval
+    @logger.debug("Starting flush cycle")
+    @files.each do |path, fd|
+      @logger.debug("Flushing file", :path => path, :fd => fd)
+      fd.flush
+    end
+    @last_flush_cycle = Time.now
+  end
+
+  # every 10 seconds or so (triggered by events, but if there are no events there's no point closing files anyway)
+  def close_stale_files
+    now = Time.now
+    return unless now - @last_stale_cleanup_cycle >= @stale_cleanup_interval
+    @logger.info("Starting stale files cleanup cycle", :files => @files)
+    inactive_files = @files.select { |path, file| not file.active }
+    @logger.debug("%d stale files found" % inactive_files.count, :inactive_files => inactive_files)
+    inactive_files.each do |path, file|
+      @logger.info("Closing file %s" % path)
+      file.close
+      @files.delete(path)
+    end
+    # mark all files as inactive, a call to write will mark them as active again
+    @files.each { |path, fd| fd.active = false }
+    @last_stale_cleanup_cycle = now
+  end
+
+  def recover_lease(path)
+    is_file_closed_available = @hdfs.respond_to? :isFileClosed
+    start = Time.now
+    first_retry = true
+
+    until start - Time.now > 900 # 15 minutes timeout 
+      recovered = @hdfs.recoverLease(path)
+      return true if recovered
+      # first retry is fast
+      if first_retry
+        sleep 4
+        first_retry = false
+        next
+      end
+
+      # on further retries we backoff and spin on isFileClosed in hopes of catching an early break
+      61.times do
+        return if is_closed_available and @hdfs.isFileClosed(path)
+        sleep 1
+      end
+    end
+    false
+  end
+
+  class DFSOutputStreamWrapper
+    # reflection locks java objects, so only do this once
+    if org.apache.hadoop.fs.FSDataOutputStream.instance_methods.include? :hflush
+      # hadoop 2.x uses hflush
+      FLUSH_METHOD = :hflush
+    else
+      FLUSH_METHOD = :flush
+    end
+    attr_accessor :active
+    def initialize(output_stream)
+      @output_stream = output_stream
+    end
+    def close
+      @output_stream.close
+    rescue IOException => e
+      logger.error("Failed to close file", :exception => e)
+    end
+    def flush
+      if FLUSH_METHOD == :hflush
+        @output_stream.hflush
+      else
+        @output_stream.flush
+        @output_stream.sync
+      end
+    rescue
+
+    end
+    def write(str)
+      bytes = str.to_java_bytes
+      @output_stream.write(bytes, 0, bytes.length)
+      @active = true
+    end
+  end
+end # class LogStash::Outputs::File
+

data/logstash-output-hdfs.gemspec

@@ -0,0 +1,26 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-output-hdfs'
+  s.version         = '0.2.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "$summary"
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Avishai Ish-Shalom"]
+  s.email           = 'avishai@fewbytes.com'
+  s.homepage        = "https://github.com/avishai-ish-shalom/logstash-hdfs"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
+
+  s.add_development_dependency 'logstash-devutils'
+end

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-hdfs
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Avishai Ish-Shalom
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-12-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  name: logstash
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: avishai@fewbytes.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/logstash/outputs/hdfs.rb
+- logstash-output-hdfs.gemspec
+homepage: https://github.com/avishai-ish-shalom/logstash-hdfs
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: $summary
+test_files: []