logstash-output-webhdfs 2.0.2 → 2.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5f10d1fc2a87cbe9cc0889cc3ebaab2c92edcdbf
-  data.tar.gz: 5184e5107893ca20d0a7501d43cc3df57b687a9a
+  metadata.gz: 4901ba13f6b2c802b94500c02c3a89279be136c5
+  data.tar.gz: 1520941c4b49bd8c20f44c5242ac6fb4973371ac
 SHA512:
-  metadata.gz: 7de713395fa8702505c3ee6a98813f0f2c86392f226873f9ed7e8439298e9be777da91f28a2ad1eff561f3517c4f83a31c6fa57ce2675bf3e4383eda2bc908f8
-  data.tar.gz: 3ed6e21b6184b555f6027088269ddc4b39512954c797b03112a45fffa6abe292f24eda70633ea13b851585a949350f4622a5330bb80a14e55a3870e19d3ced72
+  metadata.gz: 0a981834776bbc973527e6b15b09dbedde35520bb53379b001be64905ba8ab659a55fab8d5972e96bb49f555c1eb9ba78fab87ee65168fb3f176a78e6ea57c1d
+  data.tar.gz: 2daf6350fdeabf38323a6254157a60ed87d960c613060ce6b87ab8adfc82dc06e906b610461f80ef6a4e90f8de7302778ba1b328961499efef3b8692854042c3

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 2.0.4
+  - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
+# 2.0.3
+  - New dependency requirements for logstash-core for the 5.0 release
 ## 2.0.0
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895

data/CONTRIBUTORS

@@ -1,7 +1,16 @@
 The following is a list of people who have contributed ideas, code, bug
 reports, or in general have helped logstash along its way.
 
+Maintainers:
+* Björn Puttmann, dbap GmbH (dstore-dbap)
+
 Contributors:
+* Björn Puttmann, dbap GmbH (dstore-dbap)
+* Pier-Hugues Pellerin (ph)
+* Pere Urbón (purbon)
+* Suyog Rao (suyograo)
+* João Duarte (jsvd)
+* Shaunak Kashyap (ycombinator)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/README.md

@@ -1,5 +1,8 @@
 # Logstash Plugin
 
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-webhdfs-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-webhdfs-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/outputs/webhdfs.rb

@@ -4,13 +4,13 @@ require "logstash/outputs/base"
 require "stud/buffer"
 require "logstash/outputs/webhdfs_helper"
 
-# This plugin sends Logstash events into files in HDFS via 
+# This plugin sends Logstash events into files in HDFS via
 # the https://hadoop.apache.org/docs/r1.0.4/webhdfs.html[webhdfs] REST API.
 #
 # ==== Dependencies
 # This plugin has no dependency on jars from hadoop, thus reducing configuration and compatibility
 # problems. It uses the webhdfs gem from Kazuki Ohta and TAGOMORI Satoshi (@see: https://github.com/kzk/webhdfs).
-# Optional dependencies are zlib and snappy gem if you use the compression functionality. 
+# Optional dependencies are zlib and snappy gem if you use the compression functionality.
 #
 # ==== Operational Notes
 # If you get an error like:
@@ -58,6 +58,12 @@ class LogStash::Outputs::WebHdfs < LogStash::Outputs::Base
   # The server port for webhdfs/httpfs connections.
   config :port, :validate => :number, :default => 50070
 
+  # Standby namenode for ha hdfs.
+  config :standby_host, :validate => :string, :default => false
+
+  # Standby namenode port for ha hdfs.
+  config :standby_port, :validate => :number, :default => 50070
+
   # The Username for webhdfs.
   config :user, :validate => :string, :required => true
 
@@ -66,14 +72,6 @@ class LogStash::Outputs::WebHdfs < LogStash::Outputs::Base
   # `/user/logstash/dt=%{+YYYY-MM-dd}/%{@source_host}-%{+HH}.log`
   config :path, :validate => :string, :required => true
 
-  # The format to use when writing events to the file. This value
-  # supports any string and can include `%{name}` and other dynamic
-  # strings.
-  #
-  # If this setting is omitted, the full json representation of the
-  # event will be written as a single line.
-  config :message_format, :validate => :string
-
   # Sending data to webhdfs in x seconds intervals.
   config :idle_flush_time, :validate => :number, :default => 1
 
@@ -89,6 +87,11 @@ class LogStash::Outputs::WebHdfs < LogStash::Outputs::Base
   # Use httpfs mode if set to true, else webhdfs.
   config :use_httpfs, :validate => :boolean, :default => false
 
+  # Avoid appending to same file in multiple threads.
+  # This solves some problems with multiple logstash output threads and locked file leases in webhdfs.
+  # If this option is set to true, %{[@metadata][thread_id]} needs to be used in path config settting.
+  config :single_file_per_thread, :validate => :boolean, :default => false
+
   # Retry some known webhdfs errors. These may be caused by race conditions when appending to same file, etc.
   config :retry_known_errors, :validate => :boolean, :default => true
 
@@ -120,14 +123,34 @@ class LogStash::Outputs::WebHdfs < LogStash::Outputs::Base
     elsif @compression == "snappy"
       load_module('snappy')
     end
+    @main_namenode_failed = false
+    @standby_client = false
     @files = {}
+    # Create and test standby client if configured.
+    if @standby_host
+      @standby_client = prepare_client(@standby_host, @standby_port, @user)
+      begin
+        test_client(@standby_client)
+      rescue => e
+        logger.warn("Could not connect to standby namenode #{@standby_host}. Error: #{e.message}. Trying main webhdfs namenode.")
+      end
+    end
     @client = prepare_client(@host, @port, @user)
-    # Test client connection.
     begin
-      @client.list('/')
+      test_client(@client)
     rescue => e
-      @logger.error("Webhdfs check request failed. (namenode: #{@client.host}:#{@client.port}, Exception: #{e.message})")
-      raise
+      # If no standy host is configured, we need to exit here.
+      if not @standby_host
+        raise
+      else
+        # If a standby host is configured, try this before giving up.
+        do_failover
+      end
+    end
+    # Make sure @path contains %{[@metadata][thread_id]} format value if @single_file_per_thread is set to true.
+    if @single_file_per_thread and !@path.include? "%{[@metadata][thread_id]}"
+      @logger.error("Please set %{[@metadata][thread_id]} format value in @path if @single_file_per_thread is active.")
+      raise LogStash::ConfigurationError
     end
     buffer_initialize(
       :max_items => @flush_size,
@@ -140,7 +163,6 @@ class LogStash::Outputs::WebHdfs < LogStash::Outputs::Base
   end # def register
 
   def receive(event)
-    
     buffer_receive(event)
   end # def receive
 
@@ -149,6 +171,10 @@ class LogStash::Outputs::WebHdfs < LogStash::Outputs::Base
     newline = "\n"
     output_files = Hash.new { |hash, key| hash[key] = "" }
     events.collect do |event|
+      # Add thread_id to event metadata to be used as format value in path configuration.
+      if @single_file_per_thread
+        event['@metadata']['thread_id'] = Thread.current.object_id.to_s
+      end
       path = event.sprintf(@path)
       event_as_string = @codec.encode(event)
       event_as_string += newline unless event_as_string.end_with? newline
@@ -185,8 +211,14 @@ class LogStash::Outputs::WebHdfs < LogStash::Outputs::Base
       elsif
         @client.create(path, data)
       end
-      # Handle other write errors and retry to write max. @retry_times.
+    # Handle other write errors and retry to write max. @retry_times.
     rescue => e
+      # Handle StandbyException and do failover. Still we want to exit if write_tries >= @retry_times.
+      if @standby_client && (e.message.match(/Failed to connect to host/) || e.message.match(/StandbyException/))
+        do_failover
+        write_tries += 1
+        retry
+      end
       if write_tries < @retry_times
         @logger.warn("webhdfs write caused an exception: #{e.message}. Maybe you should increase retry_interval or reduce number of workers. Retrying...")
         sleep(@retry_interval * write_tries)
@@ -199,6 +231,14 @@ class LogStash::Outputs::WebHdfs < LogStash::Outputs::Base
     end
   end
 
+  def do_failover
+    if not @standby_client
+      return
+    end
+    @logger.warn("Got exception from #{@host}. Switching to #{@standby_host}")
+    @client, @standby_client = @standby_client, @client
+  end
+
   def close
     buffer_flush(:final => true)
   end # def close

data/lib/logstash/outputs/webhdfs_helper.rb

@@ -31,7 +31,16 @@ module LogStash
         client.retry_times = @retry_times if @retry_times
         client
       end
-
+      # Test client connection.
+      #@param client [WebHDFS] webhdfs client object.
+      def test_client(client)
+        begin
+          client.list('/')
+        rescue => e
+          @logger.error("Webhdfs check request failed. (namenode: #{client.host}:#{client.port}, Exception: #{e.message})")
+          raise
+        end
+      end
 
       # Compress data using the gzip methods.
       # @param data [String] stream of data to be compressed

data/logstash-output-webhdfs.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-webhdfs'
-  s.version         = '2.0.2'
+  s.version         = '2.0.4'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Plugin to write events to hdfs via webhdfs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -20,9 +20,9 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
   s.add_runtime_dependency 'webhdfs'
-  s.add_runtime_dependency 'snappy'
+  s.add_runtime_dependency 'snappy', "= 0.0.12"
   s.add_development_dependency 'logstash-devutils'
 
   s.add_development_dependency 'logstash-codec-line'

data/spec/integration/webhdfs_spec.rb

@@ -5,22 +5,20 @@ require 'webhdfs'
 require 'json'
 
 describe LogStash::Outputs::WebHdfs, :integration => true do
-
   let(:host) { 'localhost' }
   let(:port) { 50070 }
-  let(:user) { 'vagrant' }
+  let(:user) { 'test' }
+  let(:test_file) { '/user/' + user + '/%{host}.test' }
+  let(:hdfs_file_name) { 'user/' + user + '/localhost.test' }
 
-  let(:test_file) { "/test.file" }
+  let(:config) { { 'host' => host, 'user' => user, 'path' => test_file, 'compression' => 'none' } }
 
-  let(:event) { LogStash::Event.new('message' => 'Hello world!', 'source' => 'out of the blue',
-                                    'type' => 'generator', 'host' => 'localhost' ) }
+  subject(:plugin) { LogStash::Plugin.lookup("output", "webhdfs").new(config) }
 
-  let(:config) { { 'host' => host, 'user' => user,
-                   'path' => test_file, 'compression' => 'none' } }
+  let(:webhdfs_client) { WebHDFS::Client.new(host, port, user) }
 
-  subject { LogStash::Plugin.lookup("output", "webhdfs").new(config) }
-
-  let(:client) { WebHDFS::Client.new(host, port, user) }
+  let(:event) { LogStash::Event.new('message' => 'Hello world!', 'source' => 'out of the blue',
+                                    'type' => 'generator', 'host' => 'localhost' ) }
 
   describe "register and close" do
 
@@ -33,10 +31,10 @@ describe LogStash::Outputs::WebHdfs, :integration => true do
   describe '#write' do
 
     let(:config) { { 'host' => host, 'user' => user, 'flush_size' => 10,
-                     'path' => "/%{host}_test.log", 'compression' => 'none' } }
+                     'path' => test_file, 'compression' => 'none' } }
 
     after(:each) do
-      client.delete(test_file)
+      webhdfs_client.delete(hdfs_file_name)
     end
 
     describe "writing plain files" do
@@ -48,17 +46,18 @@ describe LogStash::Outputs::WebHdfs, :integration => true do
       end
 
       it 'should use the correct filename pattern' do
-        expect { client.read('localhost_test.log') }.to_not raise_error
+        expect { webhdfs_client.read(hdfs_file_name) }.to_not raise_error
       end
 
-      context "using the line codec" do
+      context "using the line codec without format" do
 
         let(:config) { { 'host' => host, 'user' => user, 'flush_size' => 10,
                          'path' => test_file, 'compression' => 'none', 'codec' => 'line' } }
 
         it 'should match the event data' do
-          expect(client.read(test_file).strip()).to eq(event.to_s)
+          expect(webhdfs_client.read(hdfs_file_name).strip()).to eq(event.to_s)
         end
+
       end
 
       context "using the json codec" do
@@ -68,7 +67,7 @@ describe LogStash::Outputs::WebHdfs, :integration => true do
 
 
         it 'should match the event data' do
-          expect(client.read(test_file).strip()).to eq(event.to_json)
+          expect(webhdfs_client.read(hdfs_file_name).strip()).to eq(event.to_json)
         end
 
       end
@@ -79,17 +78,18 @@ describe LogStash::Outputs::WebHdfs, :integration => true do
                          'path' => test_file, 'compression' => 'none', 'codec' => 'json' } }
 
         before(:each) do
-          client.delete(test_file)
+          webhdfs_client.delete(hdfs_file_name)
         end
 
         it 'should flush after configured idle time' do
           subject.register
           subject.receive(event)
-          expect { client.read(test_file) }.to raise_error(error=WebHDFS::FileNotFoundError)
+          expect { webhdfs_client.read(hdfs_file_name) }.to raise_error(error=WebHDFS::FileNotFoundError)
           sleep 3
-          expect { client.read(test_file) }.to_not raise_error
-          expect(client.read(test_file).strip()).to eq(event.to_json)
+          expect { webhdfs_client.read(hdfs_file_name) }.to_not raise_error
+          expect(webhdfs_client.read(hdfs_file_name).strip()).to eq(event.to_json)
         end
+
       end
 
     end
@@ -110,7 +110,7 @@ describe LogStash::Outputs::WebHdfs, :integration => true do
                          'path' => test_file, 'compression' => 'none', 'codec' => 'line' } }
 
         it 'should write some messages uncompressed' do
-          expect(client.read(test_file).lines.count).to eq(500)
+          expect(webhdfs_client.read(hdfs_file_name).lines.count).to eq(500)
         end
 
       end
@@ -121,11 +121,10 @@ describe LogStash::Outputs::WebHdfs, :integration => true do
                          'path' => test_file, 'compression' => 'gzip', 'codec' => 'line' } }
 
         it 'should write some messages gzip compressed' do
-          expect(Zlib::Inflate.new(window_bits=47).inflate(client.read("#{test_file}.gz")).lines.count ).to eq(500)
+          expect(Zlib::Inflate.new(window_bits=47).inflate(webhdfs_client.read("#{hdfs_file_name}.gz")).lines.count ).to eq(500)
+          webhdfs_client.delete("#{hdfs_file_name}.gz")
         end
       end
-
     end
-
   end
-end
+end

metadata

@@ -1,39 +1,33 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-webhdfs
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.0.4
 platform: ruby
 authors:
 - Björn Puttmann, loshkovskyi, Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2016-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
-      - !ruby/object:Gem::Version
-        version: 3.0.0
-  name: logstash-core
+        version: '1.0'
+  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: webhdfs
@@ -41,27 +35,27 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.12
   name: snappy
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.12
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
@@ -69,13 +63,13 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-line
@@ -83,13 +77,13 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-json
@@ -97,7 +91,7 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
@@ -129,12 +123,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []