logstash-output-webhdfs 3.0.2 → 3.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c13f13f2ad2d514bf729af26ee643ac543bcd0fd
-  data.tar.gz: 9cb1db9b084c09fde21b77b96ff54dc38ae245e2
+SHA256:
+  metadata.gz: dae08573b1282fafd1e124aaf6a206fe1820227ef0572d74184b6d21861bf8d2
+  data.tar.gz: 74d5265260ccdf7b329b7a4d8cef93c951d37005993328755fb95388e509a03c
 SHA512:
-  metadata.gz: 22cce8a3b643ce79ca9f72b2b921d5c3e31c86236f6cefc34491958aadd8ce78d5284de80160864fba84a254e366e8ab00853d5c5fdf9e4777f1430100b2aea2
-  data.tar.gz: 2d7b31dde2311f43720c5d8b96f141e29ce3dcfd412d907f472d81e848516e3ab59af65240590f251379d658dd6b65e4ae978b995a58d2312bc344c7ab86c912
+  metadata.gz: 58ec908b7dfc70f6a77475295d2a7e7cbc2d6c6c7bfa31ad9e179b21b68f723a0e4fca178f73588e4b79b58a896201eddd30dc24dcbafe69c72923ac06f0fc34
+  data.tar.gz: d6ebb6c0a8908e458a1dbc086f1c55eadba606d6c992e240f8775cda8bd1b135f1831eb4fc5a808521970a326dc82865370869cef5a9b31871cac3d9662c7f40

data/Gemfile

@@ -1,2 +1,11 @@
 source 'https://rubygems.org'
-gemspec
+
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/docs/index.asciidoc

@@ -0,0 +1,293 @@
+:plugin: webhdfs
+:type: output
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== Webhdfs output plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+This plugin sends Logstash events into files in HDFS via
+the https://hadoop.apache.org/docs/r1.0.4/webhdfs.html[webhdfs] REST API.
+
+==== Dependencies
+This plugin has no dependency on jars from hadoop, thus reducing configuration and compatibility
+problems. It uses the webhdfs gem from Kazuki Ohta and TAGOMORI Satoshi (@see: https://github.com/kzk/webhdfs).
+Optional dependencies are zlib and snappy gem if you use the compression functionality.
+
+==== Operational Notes
+If you get an error like:
+
+    Max write retries reached. Exception: initialize: name or service not known {:level=>:error}
+
+make sure that the hostname of your namenode is resolvable on the host running Logstash. When creating/appending
+to a file, webhdfs somtime sends a `307 TEMPORARY_REDIRECT` with the `HOSTNAME` of the machine its running on.
+
+==== Usage
+This is an example of Logstash config:
+
+[source,ruby]
+----------------------------------
+input {
+  ...
+}
+filter {
+  ...
+}
+output {
+  webhdfs {
+    host => "127.0.0.1"                 # (required)
+    port => 50070                       # (optional, default: 50070)
+    path => "/user/logstash/dt=%{+YYYY-MM-dd}/logstash-%{+HH}.log"  # (required)
+    user => "hue"                       # (required)
+  }
+}
+----------------------------------
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Webhdfs Output Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-compression>> |<<string,string>>, one of `["none", "snappy", "gzip"]`|No
+| <<plugins-{type}s-{plugin}-flush_size>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-host>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-idle_flush_time>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-kerberos_keytab>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-open_timeout>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-path>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-port>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-read_timeout>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-retry_interval>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-retry_known_errors>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-retry_times>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-single_file_per_thread>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-snappy_bufsize>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-snappy_format>> |<<string,string>>, one of `["stream", "file"]`|No
+| <<plugins-{type}s-{plugin}-ssl_cert>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-standby_host>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-standby_port>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-use_httpfs>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-use_kerberos_auth>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-use_ssl_auth>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-user>> |<<string,string>>|Yes
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+output plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-compression"]
+===== `compression` 
+
+  * Value can be any of: `none`, `snappy`, `gzip`
+  * Default value is `"none"`
+
+Compress output. One of ['none', 'snappy', 'gzip']
+
+[id="plugins-{type}s-{plugin}-flush_size"]
+===== `flush_size` 
+
+  * Value type is <<number,number>>
+  * Default value is `500`
+
+Sending data to webhdfs if event count is above, even if `store_interval_in_secs` is not reached.
+
+[id="plugins-{type}s-{plugin}-host"]
+===== `host` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The server name for webhdfs/httpfs connections.
+
+[id="plugins-{type}s-{plugin}-idle_flush_time"]
+===== `idle_flush_time` 
+
+  * Value type is <<number,number>>
+  * Default value is `1`
+
+Sending data to webhdfs in x seconds intervals.
+
+[id="plugins-{type}s-{plugin}-kerberos_keytab"]
+===== `kerberos_keytab` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Set kerberos keytab file. Note that the gssapi library needs to be available to use this.
+
+[id="plugins-{type}s-{plugin}-open_timeout"]
+===== `open_timeout` 
+
+  * Value type is <<number,number>>
+  * Default value is `30`
+
+WebHdfs open timeout, default 30s.
+
+[id="plugins-{type}s-{plugin}-path"]
+===== `path` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The path to the file to write to. Event fields can be used here,
+as well as date fields in the joda time format, e.g.:
+`/user/logstash/dt=%{+YYYY-MM-dd}/%{@source_host}-%{+HH}.log`
+
+[id="plugins-{type}s-{plugin}-port"]
+===== `port` 
+
+  * Value type is <<number,number>>
+  * Default value is `50070`
+
+The server port for webhdfs/httpfs connections.
+
+[id="plugins-{type}s-{plugin}-read_timeout"]
+===== `read_timeout` 
+
+  * Value type is <<number,number>>
+  * Default value is `30`
+
+The WebHdfs read timeout, default 30s.
+
+[id="plugins-{type}s-{plugin}-retry_interval"]
+===== `retry_interval` 
+
+  * Value type is <<number,number>>
+  * Default value is `0.5`
+
+How long should we wait between retries.
+
+[id="plugins-{type}s-{plugin}-retry_known_errors"]
+===== `retry_known_errors` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Retry some known webhdfs errors. These may be caused by race conditions when appending to same file, etc.
+
+[id="plugins-{type}s-{plugin}-retry_times"]
+===== `retry_times` 
+
+  * Value type is <<number,number>>
+  * Default value is `5`
+
+How many times should we retry. If retry_times is exceeded, an error will be logged and the event will be discarded.
+
+[id="plugins-{type}s-{plugin}-single_file_per_thread"]
+===== `single_file_per_thread` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Avoid appending to same file in multiple threads.
+This solves some problems with multiple logstash output threads and locked file leases in webhdfs.
+If this option is set to true, %{[@metadata][thread_id]} needs to be used in path config settting.
+
+[id="plugins-{type}s-{plugin}-snappy_bufsize"]
+===== `snappy_bufsize` 
+
+  * Value type is <<number,number>>
+  * Default value is `32768`
+
+Set snappy chunksize. Only neccessary for stream format. Defaults to 32k. Max is 65536
+@see http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
+
+[id="plugins-{type}s-{plugin}-snappy_format"]
+===== `snappy_format` 
+
+  * Value can be any of: `stream`, `file`
+  * Default value is `"stream"`
+
+Set snappy format. One of "stream", "file". Set to stream to be hive compatible.
+
+[id="plugins-{type}s-{plugin}-ssl_cert"]
+===== `ssl_cert` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Set ssl cert file.
+
+[id="plugins-{type}s-{plugin}-ssl_key"]
+===== `ssl_key` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Set ssl key file.
+
+[id="plugins-{type}s-{plugin}-standby_host"]
+===== `standby_host` 
+
+  * Value type is <<string,string>>
+  * Default value is `false`
+
+Standby namenode for ha hdfs.
+
+[id="plugins-{type}s-{plugin}-standby_port"]
+===== `standby_port` 
+
+  * Value type is <<number,number>>
+  * Default value is `50070`
+
+Standby namenode port for ha hdfs.
+
+[id="plugins-{type}s-{plugin}-use_httpfs"]
+===== `use_httpfs` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Use httpfs mode if set to true, else webhdfs.
+
+[id="plugins-{type}s-{plugin}-use_kerberos_auth"]
+===== `use_kerberos_auth` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Set kerberos authentication.
+
+[id="plugins-{type}s-{plugin}-use_ssl_auth"]
+===== `use_ssl_auth` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Set ssl authentication. Note that the openssl library needs to be available to use this.
+
+[id="plugins-{type}s-{plugin}-user"]
+===== `user` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The Username for webhdfs.
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/outputs/webhdfs.rb

@@ -111,6 +111,21 @@ class LogStash::Outputs::WebHdfs < LogStash::Outputs::Base
   # Set snappy format. One of "stream", "file". Set to stream to be hive compatible.
   config :snappy_format, :validate => ["stream", "file"], :default => "stream"
 
+  # Set kerberos authentication.
+  config :use_kerberos_auth, :validate => :boolean, :default => false
+
+  # Set kerberos keytab file. Note that the gssapi library needs to be available to use this.
+  config :kerberos_keytab, :validate => :string
+
+  # Set ssl authentication. Note that the openssl library needs to be available to use this.
+  config :use_ssl_auth, :validate => :boolean, :default => false
+
+  # Set ssl key file.
+  config :ssl_key, :validate => :string
+
+  # Set ssl cert file.
+  config :ssl_cert, :validate => :string
+
   ## Set codec.
   default :codec, 'line'
 
@@ -132,7 +147,7 @@ class LogStash::Outputs::WebHdfs < LogStash::Outputs::Base
       begin
         test_client(@standby_client)
       rescue => e
-        logger.warn("Could not connect to standby namenode #{@standby_host}. Error: #{e.message}. Trying main webhdfs namenode.")
+        logger.warn("Could not connect to standby namenode #{@standby_client.host}. Error: #{e.message}. Trying main webhdfs namenode.")
       end
     end
     @client = prepare_client(@host, @port, @user)
@@ -144,6 +159,7 @@ class LogStash::Outputs::WebHdfs < LogStash::Outputs::Base
         raise
       else
         # If a standby host is configured, try this before giving up.
+        logger.error("Could not connect to #{@client.host}:#{@client.port}. Error: #{e.message}")
         do_failover
       end
     end
@@ -235,7 +251,7 @@ class LogStash::Outputs::WebHdfs < LogStash::Outputs::Base
     if not @standby_client
       return
     end
-    @logger.warn("Got exception from #{@host}. Switching to #{@standby_host}")
+    @logger.warn("Failing over from #{@client.host}:#{@client.port} to #{@standby_client.host}:#{@standby_client.port}.")
     @client, @standby_client = @standby_client, @client
   end
 

data/lib/logstash/outputs/webhdfs_helper.rb

@@ -23,6 +23,17 @@ module LogStash
       # @return [WebHDFS] An setup client instance
       def prepare_client(host, port, username)
         client = WebHDFS::Client.new(host, port, username)
+        if @use_kerberos_auth
+          require 'gssapi'
+          client.kerberos = true
+          client.kerberos_keytab = @kerberos_keytab
+        end
+        if @use_ssl_auth
+          require 'openssl'
+          client.ssl = true
+          client.ssl_key = OpenSSL::PKey::RSA.new(open(@ssl_key))
+          client.ssl_cert = OpenSSL::X509::Certificate.new(open(@ssl_cert))
+        end
         client.httpfs_mode = @use_httpfs
         client.open_timeout = @open_timeout
         client.read_timeout = @read_timeout

data/logstash-output-webhdfs.gemspec

@@ -2,7 +2,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-webhdfs'
-  s.version         = '3.0.2'
+  s.version         = '3.0.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Plugin to write events to hdfs via webhdfs."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-webhdfs
 version: !ruby/object:Gem::Version
-  version: 3.0.2
+  version: 3.0.3
 platform: ruby
 authors:
 - Björn Puttmann, loshkovskyi, Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -100,7 +100,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: b.puttmann@dbap.de
 executables: []
 extensions: []
@@ -112,6 +114,7 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- docs/index.asciidoc
 - lib/logstash/outputs/webhdfs.rb
 - lib/logstash/outputs/webhdfs_helper.rb
 - logstash-output-webhdfs.gemspec
@@ -139,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.6.11
 signing_key:
 specification_version: 4
 summary: Plugin to write events to hdfs via webhdfs.