logstash-input-twitter 1.0.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f62f289e1b9bd6b6d3dee6d455261a03a9765531
-  data.tar.gz: 05f4f990d006d65b2077dbcb9a21c435f11981c0
+  metadata.gz: a0b170b14458a9666e7e7e955436bfef0fb8d552
+  data.tar.gz: e6f78010789cdc8a0cd57102e81315abbbca7920
 SHA512:
-  metadata.gz: c3e32235d0e06e317e0029b3936a2175943f00c59ebe844f901c013e25d6752cc6cb93e533d287d73807152a1a3c037954db994fe69efe2d08055d4206531ae9
-  data.tar.gz: 6ef349ece2a5a38d81e960512682cff205ea587611848f35de7d7517fb2b1903ba3edff807986e05becdcf40a9982f752ab02f8e78fcec8f4f53b5aed4ffebbd
+  metadata.gz: cda6d687bbe47db50f5b8648027f9d23dd5a62d4cc552e86fe66fe50e604976d4621cbc83196ddc009bdfa81f065fa76bb4ff1bcb1f33425b13fca54ed453dda
+  data.tar.gz: 25e7b7288b5adb62350dbebdd620298590deb8909ca68346b6e6c27878766da090b4c512f81c3c2d75be228164d580ba0d3e2e0fb3d039fd10a531509f23dd18

data/README.md

@@ -1,15 +1,15 @@
 # Logstash Plugin
 
-This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
 ## Documentation
 
-Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
 
 - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
-- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
 
 ## Need Help?
 
@@ -83,4 +83,4 @@ Programming is not a required skill. Whatever you've seen about open source and
 
 It is more important to the community that you are able to contribute.
 
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/inputs/twitter.rb

@@ -4,6 +4,7 @@ require "logstash/namespace"
 require "logstash/timestamp"
 require "logstash/util"
 require "logstash/json"
+require "stud/interval"
 
 # Read events from the twitter streaming api.
 class LogStash::Inputs::Twitter < LogStash::Inputs::Base
@@ -86,41 +87,47 @@ class LogStash::Inputs::Twitter < LogStash::Inputs::Base
     @logger.info("Starting twitter tracking", :keywords => @keywords)
     begin
       @client.filter(:track => @keywords.join(",")) do |tweet|
+        return if stop?
         if tweet.is_a?(Twitter::Tweet)
-          @logger.debug? && @logger.debug("Got tweet", :user => tweet.user.screen_name, :text => tweet.text)
-          if @full_tweet
-            event = LogStash::Event.new(LogStash::Util.stringify_symbols(tweet.to_hash))
-            event.timestamp = LogStash::Timestamp.new(tweet.created_at)
-          else
-            event = LogStash::Event.new(
-              LogStash::Event::TIMESTAMP => LogStash::Timestamp.new(tweet.created_at),
-              "message" => tweet.full_text,
-              "user" => tweet.user.screen_name,
-              "client" => tweet.source,
-              "retweeted" => tweet.retweeted?,
-              "source" => "http://twitter.com/#{tweet.user.screen_name}/status/#{tweet.id}"
-            )
-            event["in-reply-to"] = tweet.in_reply_to_status_id if tweet.reply?
-            unless tweet.urls.empty?
-              event["urls"] = tweet.urls.map(&:expanded_url).map(&:to_s)
-            end
-          end
-
-          # Work around bugs in JrJackson. The standard serializer won't work till we upgrade
-          event["in-reply-to"] = nil if event["in-reply-to"].is_a?(Twitter::NullObject)
+          event = from_tweet(tweet)
           decorate(event)
           queue << event
         end
       end # client.filter
-    rescue LogStash::ShutdownSignal
-      return
     rescue Twitter::Error::TooManyRequests => e
       @logger.warn("Twitter too many requests error, sleeping for #{e.rate_limit.reset_in}s")
-      sleep(e.rate_limit.reset_in)
+      Stud.stoppable_sleep(e.rate_limit.reset_in) { stop? }
       retry
     rescue => e
       @logger.warn("Twitter client error", :message => e.message, :exception => e, :backtrace => e.backtrace)
       retry
     end
   end # def run
+
+  private
+  def from_tweet(tweet)
+    @logger.debug? && @logger.debug("Got tweet", :user => tweet.user.screen_name, :text => tweet.text)
+    if @full_tweet
+      event = LogStash::Event.new(LogStash::Util.stringify_symbols(tweet.to_hash))
+      event.timestamp = LogStash::Timestamp.new(tweet.created_at)
+    else
+      event = LogStash::Event.new(
+        LogStash::Event::TIMESTAMP => LogStash::Timestamp.new(tweet.created_at),
+        "message" => tweet.full_text,
+        "user" => tweet.user.screen_name,
+        "client" => tweet.source,
+        "retweeted" => tweet.retweeted?,
+        "source" => "http://twitter.com/#{tweet.user.screen_name}/status/#{tweet.id}"
+      )
+      event["in-reply-to"] = tweet.in_reply_to_status_id if tweet.reply?
+      unless tweet.urls.empty?
+        event["urls"] = tweet.urls.map(&:expanded_url).map(&:to_s)
+      end
+    end
+
+    # Work around bugs in JrJackson. The standard serializer won't work till we upgrade
+    event["in-reply-to"] = nil if event["in-reply-to"].is_a?(Twitter::NullObject)
+
+    event
+  end
 end # class LogStash::Inputs::Twitter

data/logstash-input-twitter.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-twitter'
-  s.version         = '1.0.1'
+  s.version         = '2.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Read events from the twitter streaming api."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -20,8 +20,10 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency "logstash-core", "~> 2.0.0.snapshot"
   s.add_runtime_dependency 'twitter', ['5.12.0']
+  s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-codec-plain'
 end

data/spec/inputs/twitter_spec.rb

@@ -1,5 +1,29 @@
 require "logstash/devutils/rspec/spec_helper"
 require 'logstash/inputs/twitter'
+require 'twitter'
+
+class MockClient
+  def filter(options)
+    loop { yield }
+  end
+end
 
 describe LogStash::Inputs::Twitter do
+  context "when told to shutdown" do
+    before :each do
+      allow(Twitter::Streaming::Client).to receive(:new).and_return(MockClient.new)
+    end
+
+    it_behaves_like "an interruptible input plugin" do
+      let(:config) do
+        {
+          'consumer_key' => 'foo',
+          'consumer_secret' => 'foo',
+          'oauth_token' => 'foo',
+          'oauth_token_secret' => 'foo',
+          'keywords' => ['foo', 'bar']
+        }
+      end
+    end
+  end
 end

metadata

@@ -1,77 +1,103 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-twitter
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 2.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-29 00:00:00.000000000 Z
+date: 2015-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.0.0.snapshot
   name: logstash-core
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
-      - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.0.snapshot
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
+    - - '='
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 5.12.0
+  name: twitter
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: twitter
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 5.12.0
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 5.12.0
+        version: 0.0.22
+    - - <
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  name: stud
   prerelease: false
   type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+    - - <
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   name: logstash-devutils
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-codec-plain
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
 - CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
 - NOTICE.TXT
 - README.md
-- Rakefile
 - lib/logstash/inputs/twitter.rb
 - logstash-input-twitter.gemspec
 - spec/inputs/twitter_spec.rb
@@ -97,7 +123,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Read events from the twitter streaming api.

data/.gitignore

@@ -1,4 +0,0 @@
-*.gem
-Gemfile.lock
-.bundle
-vendor

data/Rakefile

@@ -1,7 +0,0 @@
-@files=[]
-
-task :default do
-  system("rake -T")
-end
-
-require "logstash/devutils/rake"