logstash-input-twitter 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c2c82687f3ddd3386c346f11cead1cd3a6bbc485
-  data.tar.gz: 8ba300f93af4f14852047a19f1187fc72f0cda80
+  metadata.gz: ded0fb06a53cdd6b628e29f8adcd942c7375da1e
+  data.tar.gz: bc9905695444a89e7d347250f6d91b84d860afec
 SHA512:
-  metadata.gz: b5f37dde1ba6376d57f4d4064b7f9ddcadaa7ada41449f6abfd80dc9a51b852981bbee2fc9566d38ad3bdc0badba1689ec6520bc801d90353fc55e22a1569fe4
-  data.tar.gz: f7bd667faadaa77c6e41f53785dc807e2764ae20c65276023ca7297d1108cf6479326501d248e778a463df89372db55a5d2dc648f3ab41795fe56c633e3f62cd
+  metadata.gz: 11ba55f9584d1bbd0d81b739b7c2a30b16f822204d72037c7b23fa8c90ad15853d9eb80dc65cf0c8d1637522b442827f7642ead1ca05d917ba30680be67301f9
+  data.tar.gz: dd86f9b3956e406e35fadd8dd7242550e0297d398ec576055547ef1171c334f2c778036b1397a5271770df0c0b739a08f87eea03ffbd228d1986d041b64aa4b4

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.2.0
+  - Upgraded the twitter gem to the last version available, 5.15.0
+  - Add proxy support, Fixes #7.
+
 ## 2.1.0
   - Add an option to fetch data from the sample endpoint.
   - Add hashtags, symbols and user_mentions as data for the non extended tweet event.

data/lib/logstash/inputs/twitter/patches.rb

@@ -0,0 +1,80 @@
+# encoding: utf-8
+require 'twitter/streaming/connection'
+require 'twitter/streaming/response'
+require 'twitter/streaming/message_parser'
+
+module LogStash
+  module Inputs
+    class TwitterPatches
+
+      def self.patch
+        verify_version
+        patch_json
+        patch_connection_stream
+        patch_request
+      end
+
+      private
+
+      def self.verify_version
+        raise("Incompatible Twitter gem version and the LogStash::Json.load") unless ::Twitter::Version.to_s == "5.15.0"
+      end
+
+      def self.patch_connection_stream
+        ::Twitter::Streaming::Connection.class_eval do
+          def stream(request, response)
+            socket = @tcp_socket_class.new(Resolv.getaddress(request.socket_host), request.socket_port)
+            socket = ssl_stream(socket) if !request.using_proxy?
+
+            request.stream(socket)
+            while body = socket.readpartial(1024) # rubocop:disable AssignmentInCondition
+              response << body
+            end
+          end
+
+          def ssl_stream(client)
+            client_context = OpenSSL::SSL::SSLContext.new
+            ssl_client     = @ssl_socket_class.new(client, client_context)
+            ssl_client.connect
+          end
+
+          def normalized_port(scheme)
+            HTTP::URI.port_mapping[scheme]
+          end
+        end
+      end
+
+      def self.patch_request
+        ::Twitter::Streaming::Client.class_eval do
+          def request(method, uri, params)
+            before_request.call
+            headers = ::Twitter::Headers.new(self, method, uri, params).request_headers
+            request = ::HTTP::Request.new(method, uri + '?' + to_url_params(params), headers, proxy)
+            response = ::Twitter::Streaming::Response.new do |data|
+              if item = ::Twitter::Streaming::MessageParser.parse(data) # rubocop:disable AssignmentInCondition
+                yield(item)
+              end
+            end
+            @connection.stream(request, response)
+          end
+        end
+      end
+
+      def self.patch_json
+        ::Twitter::Streaming::Response.module_eval do
+          def on_body(data)
+            @tokenizer.extract(data).each do |line|
+              next if line.empty?
+              begin
+                @block.call(LogStash::Json.load(line, :symbolize_keys => true))
+              rescue LogStash::Json::ParserError
+                # silently ignore json parsing errors
+              end
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/logstash/inputs/twitter.rb

@@ -5,6 +5,7 @@ require "logstash/timestamp"
 require "logstash/util"
 require "logstash/json"
 require "stud/interval"
+require "logstash/inputs/twitter/patches"
 
 # Ingest events from the Twitter Streaming API.
 class LogStash::Inputs::Twitter < LogStash::Inputs::Base
@@ -88,6 +89,15 @@ class LogStash::Inputs::Twitter < LogStash::Inputs::Base
   # Lets you ingore the retweets coming out of the Twitter API. Default => false
   config :ignore_retweets, :validate => :boolean, :default => false
 
+  # When to use a proxy to handle the connections
+  config :use_proxy, :validate => :boolean, :default => false
+
+  # Location of the proxy, by default the same machine as the one running this LS instance
+  config :proxy_address, :validate => :string, :default => "127.0.0.1"
+
+  # Port where the proxy is listening, by default 3128 (squid)
+  config :proxy_port, :validate => :number, :default => 3128
+
   def register
     require "twitter"
 
@@ -98,20 +108,8 @@ class LogStash::Inputs::Twitter < LogStash::Inputs::Base
     # monkey patch twitter gem to ignore json parsing error.
     # at the same time, use our own json parser
     # this has been tested with a specific gem version, raise if not the same
-    raise("Incompatible Twitter gem version and the LogStash::Json.load") unless Twitter::Version.to_s == "5.14.0"
-
-    Twitter::Streaming::Response.module_eval do
-      def on_body(data)
-        @tokenizer.extract(data).each do |line|
-          next if line.empty?
-          begin
-            @block.call(LogStash::Json.load(line, :symbolize_keys => true))
-          rescue LogStash::Json::ParserError
-            # silently ignore json parsing errors
-          end
-        end
-      end
-    end
+
+    LogStash::Inputs::TwitterPatches.patch
 
     @rest_client     = Twitter::REST::Client.new       { |c|  configure(c) }
     @stream_client   = Twitter::Streaming::Client.new  { |c|  configure(c) }
@@ -207,6 +205,12 @@ class LogStash::Inputs::Twitter < LogStash::Inputs::Base
     c.consumer_secret = @consumer_secret.value
     c.access_token = @oauth_token
     c.access_token_secret = @oauth_token_secret.value
+    if @use_proxy
+      c.proxy =  {
+        proxy_address: @proxy_address,
+        proxy_port: @proxy_port,
+      }
+    end
   end
 
   def build_options

data/logstash-input-twitter.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-twitter'
-  s.version         = '2.1.0'
+  s.version         = '2.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Read events from the twitter streaming api."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
-  s.add_runtime_dependency 'twitter', '5.14.0'
+  s.add_runtime_dependency 'twitter', '5.15.0'
   s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1'
 
   s.add_development_dependency 'logstash-devutils'

data/spec/inputs/twitter_spec.rb

@@ -188,6 +188,5 @@ describe LogStash::Inputs::Twitter do
       end
 
     end
-
   end
 end

data/spec/integration/twitter_spec.rb

@@ -71,5 +71,36 @@ describe LogStash::Inputs::Twitter do
       end
 
     end
+
+    context "when using a proxy" do
+      let(:config) do
+        <<-CONFIG
+       input {
+         twitter {
+            consumer_key => '#{ENV['TWITTER_CONSUMER_KEY']}'
+            consumer_secret => '#{ENV['TWITTER_CONSUMER_SECRET']}'
+            keywords => [ "London", "Barcelona" ]
+            oauth_token => '#{ENV['TWITTER_OAUTH_TOKEN']}'
+            oauth_token_secret => '#{ENV['TWITTER_OAUTH_TOKEN_SECRET']}'
+            full_tweet => true
+            use_proxy => true
+            proxy_address => '127.0.0.1'
+            proxy_port => 8123
+        }
+      }
+        CONFIG
+      end
+
+      let(:events) do
+        input(config) do |pipeline, queue|
+          3.times.collect { queue.pop }
+        end
+      end
+
+      it "receive a list of events from the twitter stream" do
+        expect(events.count).to eq(3)
+      end
+    end
   end
+
 end

data/spec/spec_helper.rb

@@ -6,7 +6,6 @@ class MockClient
   def filter(options)
     loop { yield }
   end
-
   alias_method :sample, :filter
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-twitter
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-17 00:00:00.000000000 Z
+date: 2015-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -36,12 +36,12 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.14.0
+        version: 5.15.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.14.0
+        version: 5.15.0
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -98,17 +98,18 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/logstash/inputs/twitter.rb
-- spec/spec_helper.rb
-- spec/inputs/twitter_spec.rb
-- spec/integration/twitter_spec.rb
-- logstash-input-twitter.gemspec
 - CHANGELOG.md
-- README.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
 - NOTICE.TXT
+- README.md
+- lib/logstash/inputs/twitter.rb
+- lib/logstash/inputs/twitter/patches.rb
+- logstash-input-twitter.gemspec
+- spec/inputs/twitter_spec.rb
+- spec/integration/twitter_spec.rb
+- spec/spec_helper.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -131,11 +132,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Read events from the twitter streaming api.
 test_files:
-- spec/spec_helper.rb
 - spec/inputs/twitter_spec.rb
 - spec/integration/twitter_spec.rb
+- spec/spec_helper.rb