logstash-input-twitter 4.0.2 → 4.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 96edca3547292597e67e8c70b43ea594ad506d9498e297481dc4f7abd27b34c1
-  data.tar.gz: 1509226ee168582ef578e20b72760689346b9364065fa8fad2ee0a51508c2a4e
+  metadata.gz: 90a556dabd75cd5e9260c327a7bdffa6bfe5e7fe4f80457901505953d7dc31f2
+  data.tar.gz: c83225608136218ea6b950f158fcd1983b3455c533f37a4f01d768b1e6dc03e2
 SHA512:
-  metadata.gz: ca5f8f2aedfb261093d586464a9f3ee4eabd6e6b12702dbbfaa9094ca276534a591f69e52fb62b1827691c83f26cda961424cfc7b61922c4dab2cc444a5913ce
-  data.tar.gz: 236df0131d90eb73ace9973f96dd78d6cbdb6fa8aee7ac4641338536f8d6eb7bc242e82a4d945f4527ea76c7bca20ef04bc0ec3dd022a3aa06e969ee14826a7a
+  metadata.gz: cced602c79e40db401fb8a2547fc2a4756a9680b62ce31852f953bcc7a17942076378e04e403505403152f8d751b9a61f90d643ff406aa3cbc90e23438291fbe
+  data.tar.gz: 7e2cef7a66efcd33566dab11d6e293b07a6a3458d22c66372b28f63dcec676dcba2fff6e63c5aec7f41836109afda8d4233f1a4c7e76625c4bbbd8fff71c8900

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.0.3
+ - Fix: broken proxy configuration [#69](https://github.com/logstash-plugins/logstash-input-twitter/pull/69)
+
 ## 4.0.2
 
   - Fix: user rest api call + proxy configuration [#68](https://github.com/logstash-plugins/logstash-input-twitter/pull/68)

data/lib/logstash/inputs/twitter.rb

@@ -128,17 +128,7 @@ class LogStash::Inputs::Twitter < LogStash::Inputs::Base
     @event_generation_error_count = 0
 
     begin
-      if @use_samples
-        @stream_client.sample do |tweet|
-          return if stop?
-          tweet_processor(queue, tweet)
-        end
-      else
-        @stream_client.filter(twitter_options) do |tweet|
-          return if stop?
-          tweet_processor(queue, tweet)
-        end
-      end
+      do_run(queue)
     rescue Twitter::Error::TooManyRequests => e
       sleep_for = e.rate_limit.reset_in || @rate_limit_reset_in # 5 minutes default value from config
       @logger.warn("Twitter too many requests error, sleeping for #{sleep_for}s")
@@ -151,6 +141,21 @@ class LogStash::Inputs::Twitter < LogStash::Inputs::Base
     end
   end # def run
 
+  def do_run(queue)
+    if @use_samples
+      @stream_client.sample do |tweet|
+        return if stop?
+        tweet_processor(queue, tweet)
+      end
+    else
+      @stream_client.filter(twitter_options) do |tweet|
+        return if stop?
+        tweet_processor(queue, tweet)
+      end
+    end
+  end
+  private :do_run
+
   def stop
     @stream_client = nil
   end

data/lib/logstash/inputs/twitter/patches.rb

@@ -8,16 +8,23 @@ module LogStash
       def self.patch
         verify_version
         patch_json
+        patch_http_request
       end
 
       private
 
       def self.verify_version
         raise("Incompatible Twitter gem version and the LogStash::Json.load") unless ::Twitter::Version.to_s == "6.2.0"
+        # NOTE: we're also known to work with twitter gem version 7.0 which uses http 4.x
+        raise("Incompatible HTTP gem version: #{HTTP::VERSION}") if HTTP::VERSION >= '5.0' || HTTP::VERSION < '3.0'
+        # after a major upgrade, unless CI is running integration specs, please run integration specs manually
       end
 
       def self.patch_json
         ::Twitter::Streaming::Response.module_eval do
+          unless method_defined? :on_body
+            raise "::Twitter::Streaming::Response#on_body not defined!" # patch bellow needs a review
+          end
           def on_body(data)
             @tokenizer.extract(data).each do |line|
               next if line.empty?
@@ -31,6 +38,27 @@ module LogStash
         end
       end
 
+      def self.patch_http_request
+        # NOTE: we might decide to drop the patch at some point, for now proxies such as squid
+        # are opinionated about having full URIs in the http head-line (having only /path fails
+        # with 400 Bad Request in Squid 4.10).
+        ::HTTP::Request.class_eval do
+          unless method_defined? :headline
+            raise "::HTTP::Request#headline not defined!" # patch bellow needs a review
+          end
+          def headline
+            request_uri =
+                if using_proxy? #&& !uri.https?
+                  uri.omit(:fragment)
+                else
+                  uri.request_uri
+                end
+
+            "#{verb.to_s.upcase} #{request_uri} HTTP/#{version}"
+          end
+        end
+      end
+
     end
   end
 end

data/logstash-input-twitter.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-twitter'
-  s.version         = '4.0.2'
+  s.version         = '4.0.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads events from the Twitter Streaming API"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/twitter_spec.rb

@@ -15,6 +15,8 @@ describe LogStash::Inputs::Twitter do
 
   let(:plugin) { LogStash::Inputs::Twitter.new(config) }
 
+  let(:queue) { Queue.new }
+
   describe "registration" do
 
     it "not raise error" do
@@ -70,7 +72,6 @@ describe LogStash::Inputs::Twitter do
   describe "fetching from sample" do
 
     let(:input) { LogStash::Inputs::Twitter.new(config) }
-    let(:queue) { Queue.new }
 
     let(:config) do
       {
@@ -142,8 +143,6 @@ describe LogStash::Inputs::Twitter do
 
       let(:input) { LogStash::Inputs::Twitter.new(config) }
 
-      let(:queue) { Queue.new }
-
       let(:stream_client)       { double("stream-client") }
 
       let(:options) do
@@ -212,5 +211,29 @@ describe LogStash::Inputs::Twitter do
       end
 
     end
+
+    describe "proxy" do
+
+      before(:each) do
+        plugin.register
+      end
+
+      let(:config) do
+        super.merge( 'use_proxy' => true, 'proxy_address' => '127.0.0.1', 'proxy_port' => 12345 )
+      end
+
+      let(:proxy_socket) { double('proxy-socket') }
+
+      it "sends full URI requests" do
+        allow_any_instance_of(Twitter::Streaming::Connection).
+            to receive(:new_tcp_socket).with('127.0.0.1', 12345).and_return(proxy_socket)
+        header_line = 'POST https://stream.twitter.com/1.1/statuses/filter.json?track=foo%2Cbar HTTP/1.1'
+        expect(proxy_socket).to receive(:write).with(/^#{Regexp.escape(header_line)}/).
+            and_return(3000) # let the stack assume all content sent was consumed
+        # NOTE: this is just bogus - we're really just testing the HTTP header line ...
+        expect(proxy_socket).to receive(:readpartial).and_return nil
+        plugin.send(:do_run, queue)
+      end
+    end
   end
 end

data/spec/integration/twitter_spec.rb

@@ -5,6 +5,16 @@ require_relative "../spec_helper"
 
 describe LogStash::Inputs::Twitter do
 
+  let(:events) do
+    input(config) do |_, queue|
+      event_count.times.collect { queue.pop }
+    end
+  end
+
+  let(:event_count) { 3 }
+
+  let(:event) { events.first }
+
   describe "#receive [integration]", :integration => true do
 
     context "keyword search" do
@@ -23,12 +33,6 @@ describe LogStash::Inputs::Twitter do
         CONFIG
       end
 
-      let(:events) do
-        input(config) do |pipeline, queue|
-          3.times.collect { queue.pop }
-        end
-      end
-
       it "receive a list of events from the twitter stream" do
         expect(events.count).to eq(3)
       end
@@ -49,28 +53,14 @@ describe LogStash::Inputs::Twitter do
         CONFIG
       end
 
-      let(:events) do
-        input(config) do |pipeline, queue|
-          3.times.collect { queue.pop }
-        end
-      end
-
-      let(:event) { events.first }
-
       it "receive a list of events from the twitter stream" do
         expect(events.count).to eq(3)
-      end
-
-      it "contains the hashtags" do
-        expect(event["hashtags"]).to be_truthy
-      end
 
-      it "contains the symbols" do
-        expect(event["symbols"]).to be_truthy
-      end
+        event.tap { |event| puts "sample event: #{event.to_hash}" } if $VERBOSE
 
-      it "contains the user_mentions" do
-        expect(event["user_mentions"]).to be_truthy
+        expect(event.get("hashtags")).to be_truthy
+        expect(event.get("symbols")).to be_truthy
+        expect(event.get("user_mentions")).to be_truthy
       end
 
     end
@@ -88,20 +78,15 @@ describe LogStash::Inputs::Twitter do
             full_tweet => true
             use_proxy => true
             proxy_address => '127.0.0.1'
-            proxy_port => 8123
+            proxy_port => 3128
         }
       }
         CONFIG
       end
 
-      let(:events) do
-        input(config) do |pipeline, queue|
-          3.times.collect { queue.pop }
-        end
-      end
-
       it "receive a list of events from the twitter stream" do
         expect(events.count).to eq(3)
+        event.tap { |event| puts "sample event (using proxy): #{event.to_hash}" } if $VERBOSE
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-twitter
 version: !ruby/object:Gem::Version
-  version: 4.0.2
+  version: 4.0.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-05 00:00:00.000000000 Z
+date: 2020-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement