logstash-input-twitter 3.0.2 → 3.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 03868427c8be1712775c28da42157a05ac74260b
-  data.tar.gz: 62ac75929f623f07047ca067e7b4b70f7f817f17
+  metadata.gz: ed7c67082ab095da36c316dbff0a2fb9ada5b5d0
+  data.tar.gz: 7be4fb0d61eeddd69c8976cdbc66408b8f32d932
 SHA512:
-  metadata.gz: a0ad512d28dff9ff61c6ecb5a47d5990105e8e321980f6026d5466f82703781ca63e61823ef90bc0340c2cf6b60168679fdc9f40131922a64292686893e9628a
-  data.tar.gz: a5c444b75e83916df14018f13ffeb1a63d42d9194294497ed0a31b6a91bbf19745591aea659f86596072141ad67627189a71c1d09ac20b9e28665c5de51886b1
+  metadata.gz: ed7ba479c15c31262026550cd5f734cbede0304d3d62cb335deabcb4b590e9206bb6d9fb098b3a6843bac84ce49dff0ff8007d86c43f411e677fe6d2f3437921
+  data.tar.gz: 2c96a1aa18ee749816846d6681c51c345b58a7c9820b2c00b634b0bbe4bf23c716cae0773a554c1a99576b893d9724bc7c751c88f3aa185087817af2df0c6fba

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 3.0.3
+  - Add protection for the (Java) Event when adding Twitter gem objects to the event in values,
+    fixes #40.
+  - add a specific rescue clause for any exceptions raised during event generation - these
+    should not be handled by the general rescue in the run method because this rescue assumes
+    a twitter gem upstream error and acts accordingly which is the wrong recovery action for an event generation failure
+  - refactor tests and add new integration test that publishes a tweet with known content.
+
 ## 3.0.2
   - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 

data/CONTRIBUTORS

@@ -4,6 +4,7 @@ reports, or in general have helped logstash along its way.
 Contributors:
 * Bernd Ahlers (bernd)
 * Colin Surprenant (colinsurprenant)
+* Guy Boertje (guyboertje)
 * John E. Vincent (lusis)
 * Jordan Sissel (jordansissel)
 * Pete Fritchman (fetep)

data/INTEGRATION.md

@@ -0,0 +1,33 @@
+### Attn: Logstash Team
+
+To run integration tests locally, you will need to add a file called `integration_credentials.rb` to the spec folder.
+
+This file is in gitignore.
+
+The contents should contain this:
+```
+if !defined?(ENV)
+  ENV = {}
+end
+
+ENV['PUB_TWITTER_CONSUMER_KEY'] = '...'
+ENV['PUB_TWITTER_CONSUMER_SECRET'] = '...'
+ENV['PUB_TWITTER_OAUTH_TOKEN'] = '...'
+ENV['PUB_TWITTER_OAUTH_TOKEN_SECRET'] = '...'
+
+ENV['TWITTER_CONSUMER_KEY'] = '...'
+ENV['TWITTER_CONSUMER_SECRET'] = '...'
+ENV['TWITTER_OAUTH_TOKEN'] = '...'
+ENV['TWITTER_OAUTH_TOKEN_SECRET'] = '...'
+
+ENV['CREDS_LOADED'] = 'twitter integration creds are loaded'
+```
+
+Use the information in the Elastic WIKI at DEV Logstash Twitter: Integration keys for logstash-input-twitter
+to fill in the `...` blanks.
+
+LogstashCiPub maps to 'PUB_TWITTER\*'
+
+LogstashCiSub maps to 'TWITTER\*'
+
+The spec_helper has been changed to run the integration tests as well if this file is present.

data/lib/logstash/inputs/twitter.rb

@@ -10,7 +10,7 @@ require "logstash/inputs/twitter/patches"
 # Ingest events from the Twitter Streaming API.
 class LogStash::Inputs::Twitter < LogStash::Inputs::Base
 
-  attr_reader :filter_options
+  attr_reader :filter_options, :event_generation_error_count
 
   config_name "twitter"
 
@@ -98,6 +98,11 @@ class LogStash::Inputs::Twitter < LogStash::Inputs::Base
   # Port where the proxy is listening, by default 3128 (squid)
   config :proxy_port, :validate => :number, :default => 3128
 
+  # Duration in seconds to wait before retrying a connection when twitter responds with a 429 TooManyRequests
+  # In some cases the 'x-rate-limit-reset' header is not set in the response and <error>.rate_limit.reset_in
+  # is nil. If this occurs then we use the integer specified here. The default is 5 minutes.
+  config :rate_limit_reset_in, :validate => :number, :default => 300
+
   def register
     require "twitter"
 
@@ -118,6 +123,11 @@ class LogStash::Inputs::Twitter < LogStash::Inputs::Base
 
   def run(queue)
     @logger.info("Starting twitter tracking", twitter_options.clone) # need to pass a clone as it modify this var otherwise
+
+    # keep track of the amount of non-specific errors rescued and logged - use in testing to verify no errors.
+    # this is because, as yet, we can't have rspec expectations on the logger instance.
+    @event_generation_error_count = 0
+
     begin
       if @use_samples
         @stream_client.sample do |tweet|
@@ -131,11 +141,13 @@ class LogStash::Inputs::Twitter < LogStash::Inputs::Base
         end
       end
     rescue Twitter::Error::TooManyRequests => e
-      @logger.warn("Twitter too many requests error, sleeping for #{e.rate_limit.reset_in}s")
-      Stud.stoppable_sleep(e.rate_limit.reset_in) { stop? }
+      sleep_for = e.rate_limit.reset_in || @rate_limit_reset_in # 5 minutes default value from config
+      @logger.warn("Twitter too many requests error, sleeping for #{sleep_for}s")
+      Stud.stoppable_sleep(sleep_for) { stop? }
       retry
     rescue => e
-      @logger.warn("Twitter client error", :message => e.message, :exception => e, :backtrace => e.backtrace, :options => @filter_options)
+      # if a lot of these errors begin to occur, the repeated retry will result in TooManyRequests errors trapped above.
+      @logger.warn("Twitter client error", :message => e.message, :exception => e.class.name, :backtrace => e.backtrace, :options => @filter_options)
       retry
     end
   end # def run
@@ -155,11 +167,15 @@ class LogStash::Inputs::Twitter < LogStash::Inputs::Base
   private
 
   def tweet_processor(queue, tweet)
-    if tweet.is_a?(Twitter::Tweet)
-      return if ignore?(tweet)
-      event = from_tweet(tweet)
-      decorate(event)
-      queue << event
+    if tweet.is_a?(Twitter::Tweet) && !ignore?(tweet)
+      begin
+        event = from_tweet(tweet)
+        decorate(event)
+        queue << event
+      rescue => e
+        @event_generation_error_count = @event_generation_error_count.next
+        @logger.error("Event generation error", :message => e.message, :exception => e.class.name, :backtrace => e.backtrace.take(10))
+      end
     end
   end
 
@@ -183,19 +199,20 @@ class LogStash::Inputs::Twitter < LogStash::Inputs::Base
         "source" => "http://twitter.com/#{tweet.user.screen_name}/status/#{tweet.id}"
       }
 
-      attributes["hashtags"] = tweet.hashtags
-      attributes["symbols"]  = tweet.symbols
-      attributes["user_mentions"]  = tweet.user_mentions
-
+      attributes["hashtags"] = tweet.hashtags.map{|ht| ht.attrs}
+      attributes["symbols"]  = tweet.symbols.map{|sym| sym.attrs}
+      attributes["user_mentions"]  = tweet.user_mentions.map{|um| um.attrs}
       event = LogStash::Event.new(attributes)
-      event.set("in-reply-to", tweet.in_reply_to_status_id) if tweet.reply?
+      if tweet.reply? && !tweet.in_reply_to_status_id.nil?
+        event.set("in-reply-to", tweet.in_reply_to_status_id)
+      end
       unless tweet.urls.empty?
         event.set("urls", tweet.urls.map(&:expanded_url).map(&:to_s))
       end
     end
 
     # Work around bugs in JrJackson. The standard serializer won't work till we upgrade
-    event.set("in-reply-to", nil) if event.get("in-reply-to").is_a?(Twitter::NullObject)
+    # event.set("in-reply-to", nil) if event.get("in-reply-to").is_a?(Twitter::NullObject)
 
     event
   end

data/logstash-input-twitter.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-twitter'
-  s.version         = '3.0.2'
+  s.version         = '3.0.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Read events from the twitter streaming api."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -26,4 +26,5 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'logstash-codec-plain'
+  s.add_development_dependency 'rspec-sequencing'
 end

data/spec/inputs/twitter_spec.rb

@@ -38,7 +38,7 @@ describe LogStash::Inputs::Twitter do
 
   describe "when told to shutdown" do
     before(:each) do
-      allow(Twitter::Streaming::Client).to receive(:new).and_return(MockClient.new)
+      allow(Twitter::Streaming::Client).to receive(:new).and_return(LogstashTwitterInput::MockClient.new)
     end
     it_behaves_like "an interruptible input plugin"
   end
@@ -67,7 +67,7 @@ describe LogStash::Inputs::Twitter do
 
     it "uses the sample endpoint" do
       expect(stream_client).to receive(:sample).once
-      run_input_with(input, queue)
+      LogstashTwitterInput.run_input_with(input, queue)
     end
 
   end
@@ -133,7 +133,7 @@ describe LogStash::Inputs::Twitter do
 
       it "using the filter endpoint" do
         expect(stream_client).to receive(:filter).with(options).once
-        run_input_with(input, queue)
+        LogstashTwitterInput.run_input_with(input, queue)
       end
 
       context "when not filtering retweets" do
@@ -158,7 +158,7 @@ describe LogStash::Inputs::Twitter do
           expect(input).to receive(:from_tweet).with(tweet)
           expect(stream_client).to receive(:filter).at_least(:once).and_yield(tweet)
           expect(queue).to receive(:<<)
-          run_input_with(input, queue)
+          LogstashTwitterInput.run_input_with(input, queue)
         end
       end
 
@@ -183,7 +183,7 @@ describe LogStash::Inputs::Twitter do
           allow(tweet).to receive(:retweet?).and_return(true)
           expect(stream_client).to receive(:filter).at_least(:once).and_yield(tweet)
           expect(queue).not_to receive(:<<)
-          run_input_with(input, queue)
+          LogstashTwitterInput.run_input_with(input, queue)
         end
       end
 

data/spec/integration/twitter_pubsub_spec.rb

@@ -0,0 +1,86 @@
+require_relative "../spec_helper"
+
+Thread.abort_on_exception = true
+
+describe LogStash::Inputs::Twitter do
+  describe "full integration", :integration => true do
+    let(:pub_config) do
+      {
+        :consumer_key => ENV['PUB_TWITTER_CONSUMER_KEY'],
+        :consumer_secret => ENV['PUB_TWITTER_CONSUMER_SECRET'],
+        :access_token => ENV['PUB_TWITTER_OAUTH_TOKEN'],
+        :access_token_secret => ENV['PUB_TWITTER_OAUTH_TOKEN_SECRET']
+      }
+    end
+
+    let(:publisher) do
+      Twitter::REST::Client.new(pub_config)
+    end
+
+    let(:plugin_cfg) do
+      {
+        'consumer_key' => ENV['TWITTER_CONSUMER_KEY'],
+        'consumer_secret' => ENV['TWITTER_CONSUMER_SECRET'],
+        'oauth_token' => ENV['TWITTER_OAUTH_TOKEN'],
+        'oauth_token_secret' => ENV['TWITTER_OAUTH_TOKEN_SECRET'],
+        'keywords' => ['logstash_ci_publish'],
+        'full_tweet' => full_tweets
+      }
+    end
+
+    let(:plugin) { described_class.new(plugin_cfg) }
+    let(:queue)  { Array.new }
+    let(:tweet_count) { 1 }
+    let(:full_tweets) { false }
+
+    before do
+      Thread.new do
+        plugin.register
+        plugin.run(queue)
+      end
+    end
+
+    context "partial tweet pub sub" do
+      it "receives an event from the twitter stream" do
+        RSpec::Sequencing.run_after(1, "update tweet status") do
+          twt = "logstash_ci_publish partial tweet test #{Time.now} #{SecureRandom.hex(6)} $AAPL #lscipub @logstash https://www.elastic.co/downloads/logstash"
+          publisher.update_with_media(twt, File.open(LogstashTwitterInput.fixture("small_smile.png")))
+        end.then_after(3, "stop plugin") do
+          plugin.stop
+          true
+        end.value
+        expect(queue.size).to eq(tweet_count)
+        expect(plugin.event_generation_error_count).to eq(0)
+        event = queue.first
+        expect { event.to_json }.not_to raise_error
+        expect(event.get("hashtags").size).to be > 0
+        expect(event.get("symbols").size).to be > 0
+        expect(event.get("user_mentions").size).to be > 0
+        expect(event.get("urls").size).to be > 0
+      end
+    end
+
+    context "full tweet pub sub" do
+      let(:full_tweets) { true }
+
+      it "receives an event from the twitter stream" do
+        RSpec::Sequencing.run_after(0.1, "update tweet status") do
+          twt = "logstash_ci_publish full tweet test #{Time.now} #{SecureRandom.hex(6)} $AAPL #lscipub @logstash https://www.elastic.co/downloads/logstash"
+          publisher.update_with_media(twt, File.open(LogstashTwitterInput.fixture("small_smile.png")))
+        end.then_after(3, "stop plugin") do
+          plugin.stop
+          true
+        end.value
+        expect(queue.count).to eq(tweet_count)
+        expect(plugin.event_generation_error_count).to eq(0)
+        event = queue.first
+        expect { LogStash::Json.dump(event.to_hash) }.not_to raise_error
+        expect(event.get("[entities][hashtags]").size).to be > 0
+        expect(event.get("[entities][symbols]").size).to be > 0
+        expect(event.get("[entities][user_mentions]").size).to be > 0
+        expect(event.get("[entities][urls]").size).to be > 0
+        expect(event.get("[extended_tweet][entities][media]").size).to be > 0
+      end
+    end
+  end
+end

data/spec/integration/twitter_spec.rb

@@ -1,8 +1,11 @@
+__END__
+# this test is quite unreliable - commenting it out
+
 require_relative "../spec_helper"
 
 describe LogStash::Inputs::Twitter do
 
-  describe "#receive", :integration => true do
+  describe "#receive [integration]", :integration => true do
 
     context "keyword search" do
       let(:config) do
@@ -13,7 +16,7 @@ describe LogStash::Inputs::Twitter do
             consumer_secret => '#{ENV['TWITTER_CONSUMER_SECRET']}'
             oauth_token => '#{ENV['TWITTER_OAUTH_TOKEN']}'
             oauth_token_secret => '#{ENV['TWITTER_OAUTH_TOKEN_SECRET']}'
-            keywords => [ 'Barcelona' ]
+            keywords => [ 'London', 'Politics', 'New York', 'Samsung', 'Apple' ]
             full_tweet => true
         }
       }
@@ -103,4 +106,4 @@ describe LogStash::Inputs::Twitter do
     end
   end
 
-end
+end

data/spec/spec_helper.rb

@@ -1,18 +1,40 @@
-require "logstash/devutils/rspec/spec_helper"
+require 'logstash/devutils/rspec/spec_helper'
 require 'logstash/inputs/twitter'
 require 'twitter'
+require 'rspec_sequencing'
 
-class MockClient
-  def filter(options)
-    loop { yield }
+creds_file_path = File.expand_path('spec/integration_credentials.rb')
+if File.exist?(creds_file_path)
+  load creds_file_path
+  RSpec.configure do |config|
+    # enable integrations only when credentials are loaded
+    exclusions = config.exclusion_filter
+    exclusions.delete(:integration)
+    config.exclusion_filter = exclusions
   end
-  alias_method :sample, :filter
 end
 
-def run_input_with(input, queue)
-  t = Thread.new(input, queue) do |_input, _queue|
-    _input.run(_queue)
+module LogstashTwitterInput
+  class MockClient
+    def filter(options)
+      loop { yield }
+    end
+    alias_method :sample, :filter
   end
-  sleep 0.1
-  t.kill
-end
+
+  def self.run_input_with(input, queue)
+    t = Thread.new(input, queue) do |_input, _queue|
+      _input.run(_queue)
+    end
+    sleep 0.1
+    t.kill
+  end
+
+  def self.fixture_path
+    File.expand_path('../fixtures', __FILE__)
+  end
+
+  def self.fixture(file)
+    File.new(self.fixture_path + '/' + file)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-twitter
 version: !ruby/object:Gem::Version
-  version: 3.0.2
+  version: 3.0.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2016-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -92,6 +92,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rspec-sequencing
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -101,13 +115,16 @@ files:
 - CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
+- INTEGRATION.md
 - LICENSE
 - NOTICE.TXT
 - README.md
 - lib/logstash/inputs/twitter.rb
 - lib/logstash/inputs/twitter/patches.rb
 - logstash-input-twitter.gemspec
+- spec/fixtures/small_smile.png
 - spec/inputs/twitter_spec.rb
+- spec/integration/twitter_pubsub_spec.rb
 - spec/integration/twitter_spec.rb
 - spec/spec_helper.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
@@ -132,11 +149,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Read events from the twitter streaming api.
 test_files:
+- spec/fixtures/small_smile.png
 - spec/inputs/twitter_spec.rb
+- spec/integration/twitter_pubsub_spec.rb
 - spec/integration/twitter_spec.rb
 - spec/spec_helper.rb