gnip_api 1.2.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 515a1f5c48c72c4f23dfc7e4882d6f7b8b47067c
-  data.tar.gz: f4775d00111fb0811aefb798aa178357927ccf02
+  metadata.gz: 5beeb3c15e9e6e05694dbcb5e5ddc8bce5985659
+  data.tar.gz: 36c3b182f4d95e503c0963a0859c6d3b3c57a4ba
 SHA512:
-  metadata.gz: 358bd8b2aba8653f782a747793b1f4580712ef74c3b188ef9a5fd65dfb8cd5cfeab1d8238327a147b42a9226abc925bab2999bb83c74f929abb917fe0a9c0ebd
-  data.tar.gz: 89bd54df5d5e31a04f467996e104651c4ace873de6748a96d33788eaf89e01d022aa26f3c81f8261998a138346f9ebcdc192ca2c5f5426267cbc7502bad0df10
+  metadata.gz: cac268d5f61308340ad2f391f007684e4b0276eea9202dce3dc61d8871b81bc8ed6b2d62157617a3709446e9dd2fd661e4fffd6d44e818d1526d0017b2f5f869
+  data.tar.gz: ed8da7a8e75e2d5f68ea75c95d59a7598cf37427c39862a5dd0027e8bda8f80233b21e0ecffe0691b84656852f93bf5503ffab2ff74a924bf3c7a103cc3797a5

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gnip_api (1.2.0)
+    gnip_api (1.2.1)
       addressable
       httparty
       yajl-ruby

data/lib/gnip_api.rb

@@ -4,6 +4,7 @@ require 'yajl'
 require 'logger'
 require 'addressable/uri'
 require 'uri'
+require 'pty'
 
 require "gnip_api/version"
 require "gnip_api/errors"

data/lib/gnip_api/adapter.rb

@@ -1,4 +1,33 @@
 module GnipApi
+  # The interface to the world. Wraps the net adapter of choice and exposes common methods
+  # for the rest of the classes to use, abstracting details of the request/response.
+  # A note on the data:
+  # HTTParty was selected over Excon for the simple reason that Excon doesn't handle
+  # GZip responses. HTTParty uses net/http under the hood and handles this as expected.
+  # However, for reasons I don't know at this point, the Zlib responsible of inflating
+  # the GZip compressed data waits to fill in a buffer in order to return a decompressed
+  # chunk. You can test this on a farily busy stream and you'll see that all chunks you
+  # receive decompressed are almsot always 16384 bytes. Additionally, the \r\n Gnip sends
+  # as a keep alive signal seems to be buffered further.
+  # As a result I expose 2 additional methods for consuming the stream that uses Curl
+  # instead. Curl handles the GZip data better. You can launch it in a shell and the 
+  # keep alive signals are properly there.
+  # These alternatives are a bit of a cheat, but allows delegation of handling the actual
+  # connection and the GZip data.
+  # 
+  # #stream_get will work as usual, using HTTParty and Net::HTTP stuff.
+  #
+  # #io_curl_stream will launch a subprocess and capture output with IO.popen. This method
+  # behaves similarly to #stream_get regarding the keep alive signals. However, these are
+  # returned all at once when a bigger chunk of data is sent over the stream. Still, it reads
+  # them well and can be handled.
+  #
+  # #pty_curl_stream is different. Using the PTY features, a shell is opened and the output 
+  # captured is as (in my opinion) should be. All keep-alive signals are returned when received
+  # with no wait at all.
+  #
+  # All this will remain experimental until tested on a real environment. I recomment to use
+  # #stream_get which was already tested and works as expected.
   class Adapter
     GET = 'GET'
     POST = 'POST'
@@ -13,30 +42,21 @@ module GnipApi
     end
 
     def get request
-      request.log!
-      data = HTTParty.get request.uri, :basic_auth => auth, :timeout => default_timeout
-      response = create_response(request, data.code, data.body, data.headers)
-      response.check_for_errors!
-      return response.body unless response.body.empty?
-      return true
+      options = default_options(request)
+      data = HTTParty.get request.uri, options
+      create_response(request, data.code, data.body, data.headers)
     end
 
     def post request
-      request.log!
-      data = HTTParty.post request.uri, :basic_auth => auth, :body => request.payload, :timeout => default_timeout
-      response = create_response(request, data.code, data.body, data.headers)
-      response.check_for_errors!
-      return response.body unless response.body.empty?
-      return true
+      options = default_options(request)
+      data = HTTParty.post request.uri, options
+      create_response(request, data.code, data.body, data.headers)
     end
 
     def delete request
-      request.log!
-      data = HTTParty.post request.uri, :basic_auth => auth, :body => request.payload, :timeout => default_timeout
-      response = create_response(request, data.code, data.body, data.headers)
-      response.check_for_errors!
-      return response.body unless response.body.empty?
-      return true
+      options = default_options(request)
+      data = HTTParty.post request.uri, options
+      create_response(request, data.code, data.body, data.headers)
     end
 
     def stream_get request
@@ -46,12 +66,63 @@ module GnipApi
           yield(data)
         end
       rescue Zlib::BufError => error
-        GnipApi.config.logger.error "STREAM ERROR -> #{error.class} -- #{error.message}\n" + error.backtrace.join("\n")
+        logger.error "STREAM ERROR -> #{error.class} -- #{error.message}\n" + error.backtrace.join("\n")
         raise error
       end
     end
 
+    def io_curl_stream request
+      request.log!
+      auth_header = "Authorization: Basic #{base64_auth}"
+      cmd = "curl --compressed -s --header \"#{auth_header}\" \"#{request.uri}\""
+      begin
+        IO.popen(cmd) do |io|
+          while line = io.gets.strip do
+            logger.info "Keep alive received" if line == ''
+            next if line == ''
+            yield(line)
+          end
+        end
+      ensure
+        logger.warn "Stream closed"
+      end
+    end
+
+    def pty_curl_stream request
+      request.log!
+      auth_header = "Authorization: Basic #{base64_auth}"
+      cmd = "curl --compressed -s --header \"#{auth_header}\" \"#{request.uri}\""
+      begin
+        PTY.spawn(cmd) do |stdout, stdin, pid|
+          begin
+            stdout.each do |line|
+              logger.info "Keep alive received" if line.strip == ''
+              next if line.strip == ''
+              yield(line.strip)
+            end
+          rescue Errno::EIO
+          end
+        end
+      rescue PTY::ChildExited
+        logger.warn "Stream closed"
+      end
+    end
+
     private
+    def default_options request
+      {
+        :headers => request.headers,
+        :basic_auth => auth,
+        :timeout => default_timeout,
+        :debug_output => (debug ? logger : nil),
+        :body => request.payload
+      }.delete_if{|k,v| v.nil? || v == ''}
+    end
+
+    def base64_auth
+      Base64.encode64("#{auth[:username]}:#{auth[:password]}").strip
+    end
+
     def auth
       {
         :username => username,

data/lib/gnip_api/endpoints.rb

@@ -1,31 +1,35 @@
 module GnipApi
   class Endpoints
     class << self
-      def powertrack_rules label
+      def powertrack_rule_validator
+        URI("https://gnip-api.twitter.com/rules/powertrack/accounts/#{account}/publishers/twitter/#{label}/validation.json")
+      end
+
+      def powertrack_rules
         URI("https://gnip-api.twitter.com/rules/powertrack/accounts/#{account}/publishers/twitter/#{label}.json")
       end
 
-      def powertrack_stream label
+      def powertrack_stream
         URI("https://gnip-stream.twitter.com/stream/powertrack/accounts/#{account}/publishers/twitter/#{label}.json")
       end
 
-      def powertrack_rule_validator label
+      def powertrack_rule_validator
         URI("https://gnip-api.twitter.com/rules/powertrack/accounts/#{account}/publishers/twitter/#{label}/validation.json")
       end
 
-      def powertrack_stream_replay label
+      def powertrack_stream_replay
         URI("https://gnip-stream.twitter.com/replay/powertrack/accounts/#{account}/publishers/twitter/#{label}.json")
       end
 
-      def powertrack_rule_replay label
+      def powertrack_rule_replay
         URI("https://gnip-api.twitter.com/rules/powertrack-replay/accounts/#{account}/publishers/twitter/#{label}.json")
       end
 
-      def search_activities label
+      def search_activities
         URI("https://gnip-api.twitter.com/search/fullarchive/accounts/#{account}/#{label}.json")
       end
       
-      def search_counts label
+      def search_counts
         URI("https://gnip-api.twitter.com/search/fullarchive/accounts/#{account}/#{label}/counts.json")
       end
 
@@ -33,6 +37,10 @@ module GnipApi
       def account
         GnipApi.configuration.account
       end
+
+      def label
+        GnipApi.configuration.label
+      end
     end
   end
 end

data/lib/gnip_api/errors.rb

@@ -14,11 +14,7 @@ module GnipApi
     end
 
     module Adapter
-      class RequestError < StandardError
-        def initialize msg='Request failed'
-          @message = msg
-        end
-      end
+      class RequestError < StandardError; end
 
       class RateLimitError < StandardError
         def initialize msg='Exceded rate limits'

data/lib/gnip_api/power_track/rules.rb

@@ -5,29 +5,19 @@
 module GnipApi
   module PowerTrack
     class Rules
-      attr_reader :adapter
-
-      # In order to do any operation, you need to specify:
-      # - label: the label of your stream
-      # - source: which data source to use (I think only twitter is available)
-      def initialize params={}
-        @adapter = GnipApi::Adapter.new
-        @label = params[:label] || GnipApi.config.label
-      end
-
       # Returns an array of defined rules
       def list
         request = create_get_request
-        rules = adapter.get(request)
+        rules = fetch_data(request)
         parse_rules(rules)
       end
 
       # Creates the specified rule. Parameters:
       # - rules: GnipApi::PowerTrack::Rule object
       def create rules
-        raise GnipApi::Errors::PowerTrack::MissingRules.new if rules.nil? || rules.empty?
+        raise ArgumentError.new('No rules provided') if rules.nil? || rules.empty?
         request = create_post_request(construct_rules(rules))
-        response = adapter.post(request)
+        response = fetch_data(request)
         return true if response.nil?
         return GnipApi::JsonParser.new.parse(response)
       end
@@ -35,9 +25,18 @@ module GnipApi
       # Deletes the specified rule. Parameters:
       # - rules: GnipApi::PowerTrack::Rule object
       def delete rules
-        raise GnipApi::Errors::PowerTrack::MissingRules.new if rules.nil? || rules.empty?
+        raise ArgumentError.new('No rules provided') if rules.nil? || rules.empty?
         request = create_delete_request(construct_rules(rules))
-        response = adapter.delete(request)
+        response = fetch_data(request)
+        return true if response.nil?
+        return GnipApi::JsonParser.new.parse(response)
+      end
+
+      def validate rules
+        raise ArgumentError.new('No rules provided') if rules.nil? || rules.empty?
+        byebug
+        request = create_validation_request(construct_rules(rules))
+        response = fetch_data(request)
         return true if response.nil?
         return GnipApi::JsonParser.new.parse(response)
       end
@@ -58,8 +57,20 @@ module GnipApi
       end
 
       private
+      def fetch_data(request)
+        request.execute!
+      end
+
       def endpoint
-        GnipApi::Endpoints.powertrack_rules(@label)
+        GnipApi::Endpoints.powertrack_rules
+      end
+
+      def validation_endpoint
+        GnipApi::Endpoints.powertrack_rule_validator
+      end
+
+      def create_validation_request payload
+        GnipApi::Request.new_post(validation_endpoint, payload)
       end
 
       def create_get_request
@@ -73,7 +84,7 @@ module GnipApi
       def create_delete_request payload
         delete_url = endpoint
         delete_url.query = '_method=delete'
-        GnipApi::Request.new_delete(delete_url, payload)
+        GnipApi::Request.new_post(delete_url, payload)
       end
 
     end

data/lib/gnip_api/power_track/stream.rb

@@ -1,13 +1,10 @@
 module GnipApi
   module PowerTrack
     class Stream
-      attr_reader :adapter
-      
-      def initialize params = {}
-        @stream = params[:stream] || GnipApi.config.label
-        @user = params[:user] || GnipApi.configuration.user
-        @password = params[:password] || GnipApi.configuration.password
-        @account = params[:account] || GnipApi.configuration.account
+      def initialize
+        @user = GnipApi.configuration.user
+        @password = GnipApi.configuration.password
+        @account = GnipApi.configuration.account
         @adapter = GnipApi::Adapter.new
         @buffer = GnipApi::PowerTrack::Buffer.new
         @running = false
@@ -41,21 +38,100 @@ module GnipApi
         end
       end
 
-      def consume
-        read_stream do |data|
-          yield(process_entries(data))
+      # The following methods are different ways of consuming the stream
+      # There are 3 different methods that return data slighly different.
+      # :common method uses a simple HTTParty request reading chunks and 
+      # decoding the GZip. This method has a flaw that it waits for certain
+      # data to be buffered by Zlib in order to return a decoded chunk.
+      # :common will return chunks that may contain more than 1 objects.
+      # 
+      # :io method uses curl under the hood, in combination with IO.popen
+      # to captrue stdout. For this method a single line is returned, which
+      # would be an object sent to stream. Curl handles the GZip decoding 
+      # better, however the read method for the IO buffers up the keep alive
+      # signals due to not flushing STDOUT.
+      #
+      # :pty method is an alternative for :io in where the stdout output
+      # is captured as it comes using PTY features. It almost works the 
+      # same as :io, but the keep alive signals are now captured properly.
+      def consume stream_method=:common
+        raise ArgumentError, "Block required, non given" unless block_given?
+        if stream_method == :common
+          read_stream do |data|
+            yield(process_entries(data))
+          end
+        elsif stream_method == :io
+          read_io_stream do |data|
+            yield(process_entries([data]))
+          end
+        elsif stream_method == :pty
+          read_pty_stream do |data|
+            yield(process_entries([data]))
+          end
+        else 
+          raise ArgumentError, "Undefined stream method #{stream_method}"
         end
       end 
 
-      def consume_raw
-        read_stream do |data|
-          yield(data)
+      def consume_raw stream_method=:common
+        raise ArgumentError, "Block required, non given" unless block_given?
+        if stream_method == :common
+          read_stream do |data|
+            yield(data)
+          end
+        elsif stream_method == :io
+          read_io_stream do |data|
+            yield(data)
+          end
+        elsif stream_method == :pty
+          read_pty_stream do |data|
+            yield(data)
+          end
+        else 
+          raise ArgumentError, "Undefined stream method #{stream_method}"
         end
       end 
 
-      def consume_json
-        read_stream do |data|
-          yield(data.map{|item| parse_json(item)})
+      def consume_json stream_method=:common
+        raise ArgumentError, "Block required, non given" unless block_given?
+        if stream_method == :common
+          read_stream do |data|
+            yield(data.map{|item| parse_json(item)})
+          end
+        elsif stream_method == :io
+          read_io_stream do |data|
+            yield(parse_json(data))
+          end
+        elsif stream_method == :pty
+          read_pty_stream do |data|
+            yield(parse_json(data))
+          end
+        else
+          raise ArgumentError, "Undefined stream method #{stream_method}"
+        end
+      end
+
+      def read_io_stream
+        request = create_request
+        logger.info "Opening PowerTrack parsed stream"
+        begin
+          @adapter.io_curl_stream(request) do |data|
+            yield data
+          end
+        ensure
+          logger.warn "Closing stream"
+        end
+      end
+
+      def read_pty_stream
+        request = create_request
+        logger.info "Opening PowerTrack parsed stream"
+        begin
+          @adapter.pty_curl_stream(request) do |data|
+            yield data
+          end
+        ensure
+          logger.warn "Closing stream"
         end
       end
 
@@ -63,8 +139,8 @@ module GnipApi
         request = create_request
         logger.info "Opening PowerTrack parsed stream"
         begin
-          adapter.stream_get request do |chunk|
-            stream_running!
+          @adapter.stream_get request do |chunk|
+            stream_running!(@buffer, chunk)
             @buffer.insert! chunk
             yield @buffer.read! if block_given?
           end
@@ -114,7 +190,7 @@ module GnipApi
       end
 
       def endpoint
-        GnipApi::Endpoints.powertrack_stream(@stream)
+        GnipApi::Endpoints.powertrack_stream
       end
 
     end

data/lib/gnip_api/request.rb

@@ -21,6 +21,24 @@ module GnipApi
       @payload = params[:payload]
       @headers = params[:headers]
       @request_method = params[:request_method]
+      @adapter = GnipApi::Adapter.new
+    end
+
+    def execute!
+      log!
+      case request_method
+      when GnipApi::Adapter::GET
+        response = @adapter.get(self)
+      when GnipApi::Adapter::POST
+        response = @adapter.post(self)
+      when GnipApi::Adapter::DELETE
+        response = @adapter.delete(self)
+      else
+        raise 'RequestNotAllowed'
+      end
+      response.check_for_errors!
+      return response.body unless response.body.empty?
+      return true
     end
 
     def log!
@@ -28,6 +46,6 @@ module GnipApi
       GnipApi.logger.debug "Headers -> #{headers.inspect}"
       GnipApi.logger.debug "Payload -> #{payload.inspect}"
     end
-    
+
   end
 end

data/lib/gnip_api/response.rb

@@ -48,7 +48,7 @@ module GnipApi
         GnipApi.logger.debug "Request payload -> #{request.payload.inspect}"
         raise GnipApi::Errors::Adapter::GnipSoftwareError.new error_message if status == 503
         raise GnipApi::Errors::Adapter::RateLimitError.new error_message if status == 429
-        raise GnipApi::Errors::Adapter::RequestError.new error_message
+        raise GnipApi::Errors::Adapter::RequestError.new("Status #{status} #{error_message}")
       end
     end
 

data/lib/gnip_api/search.rb

@@ -4,18 +4,11 @@
 
 module GnipApi
   class Search
-    attr_reader :adapter
-
-    def initialize params={}
-      @adapter = GnipApi::Adapter.new
-      @label = params[:label] || GnipApi.config.label
-    end
-
     def activities options={}
       required_options?(options)
       payload = construct_activities_payload(options)
       request = GnipApi::Request.new_post(activities_endpoint, payload)
-      data = adapter.post(request)
+      data = fetch_data(request)
       return parse_activities_response(data)
     end
 
@@ -23,17 +16,21 @@ module GnipApi
       required_options?(options)
       payload = construct_counts_payload(options)
       request = GnipApi::Request.new_post(count_endpoint, payload)
-      data = adapter.post(request)
+      data = fetch_data(request)
       return parse_counts_response(data)
     end
 
     private
+    def fetch_data(request)
+      request.execute!
+    end
+
     def count_endpoint
-      GnipApi::Endpoints.search_counts(@label)
+      GnipApi::Endpoints.search_counts
     end
 
     def activities_endpoint
-      GnipApi::Endpoints.search_activities(@label)
+      GnipApi::Endpoints.search_activities
     end
 
     def required_options

data/lib/gnip_api/version.rb

@@ -1,3 +1,3 @@
 module GnipApi
-  VERSION = "1.2.0"
+  VERSION = "1.2.1"
 end

data/spec/gnip_api/endpoints_spec.rb

@@ -3,22 +3,20 @@ require 'spec_helper'
 describe GnipApi::Endpoints do
   before do
     configure_gem
-    @source = 'somewhere'
-    @label = 'stream'
   end
   
   describe '.powertrack_rules' do
     it 'returns URI object' do
-      expect(GnipApi::Endpoints.powertrack_rules(@label).kind_of?(URI)).to eq(true)
+      expect(GnipApi::Endpoints.powertrack_rules.kind_of?(URI)).to eq(true)
     end
 
     describe 'URI returned' do
       before do
-        @uri = GnipApi::Endpoints.powertrack_rules(@label)
+        @uri = GnipApi::Endpoints.powertrack_rules
       end
 
       it 'contains label in path' do
-        expect(@uri.path.include?(@label)).to eq(true)
+        expect(@uri.path.include?(GnipApi.configuration.label)).to eq(true)
       end
 
       it 'includes account in path' do
@@ -29,16 +27,16 @@ describe GnipApi::Endpoints do
 
   describe '.powertrack_stream' do
     it 'returns URI object' do
-      expect(GnipApi::Endpoints.powertrack_stream(@label).kind_of?(URI)).to eq(true)
+      expect(GnipApi::Endpoints.powertrack_stream.kind_of?(URI)).to eq(true)
     end
 
     describe 'URI returned' do
       before do
-        @uri = GnipApi::Endpoints.powertrack_stream(@label)
+        @uri = GnipApi::Endpoints.powertrack_stream
       end
 
       it 'contains label in path' do
-        expect(@uri.path.include?(@label)).to eq(true)
+        expect(@uri.path.include?(GnipApi.configuration.label)).to eq(true)
       end
 
       it 'includes account in path' do

data/spec/gnip_api/power_track/rules_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 describe GnipApi::PowerTrack::Rules do
   before do
-    @api = GnipApi::PowerTrack::Rules.new(:source => 'source', :label => 'label')
+    @api = GnipApi::PowerTrack::Rules.new
     @rules = [GnipApi::PowerTrack::Rule.new(:value => 'r1', :tag => 't1'),
               GnipApi::PowerTrack::Rule.new(:value => 'r2')]
     @json = {:rules => [{:value => 'r1', :tag => 't1'}, {:value => 'r2'}]}.to_json
@@ -23,19 +23,19 @@ describe GnipApi::PowerTrack::Rules do
 
   describe '#delete' do
     it 'raises error if no rules passed' do
-      expect(Proc.new{@api.delete([])}).to raise_error(GnipApi::Errors::PowerTrack::MissingRules)
+      expect(Proc.new{@api.delete([])}).to raise_error(ArgumentError)
     end
   end
 
   describe '#create' do
     it 'raises error if no rules passed' do
-      expect(Proc.new{@api.create([])}).to raise_error(GnipApi::Errors::PowerTrack::MissingRules)
+      expect(Proc.new{@api.create([])}).to raise_error(ArgumentError)
     end
   end
 
   describe '#list' do
     before do
-      expect(@api.adapter).to receive(:get).and_return(@json)
+      allow(@api).to receive(:fetch_data).and_return(@json)
     end
 
     it 'returns an array' do

data/spec/gnip_api/search_spec.rb

@@ -13,7 +13,7 @@ describe GnipApi::Search do
     before do
       @raw_response = File.read(fixture_path.join('search_api/search_counts_response.json'))
       @api = GnipApi::Search.new
-      allow(@api.adapter).to receive(:post).and_return(@raw_response)
+      allow(@api).to receive(:fetch_data).and_return(@raw_response)
     end
 
     it 'raises error if missing params' do
@@ -66,7 +66,7 @@ describe GnipApi::Search do
     before do
       @raw_response = File.read(fixture_path.join('search_api/search_activities_response.json'))
       @api = GnipApi::Search.new
-      allow(@api.adapter).to receive(:post).and_return(@raw_response)
+      allow(@api).to receive(:fetch_data).and_return(@raw_response)
     end
 
     it 'raises error if missing params' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gnip_api
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Rayko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-09 00:00:00.000000000 Z
+date: 2017-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -182,7 +182,6 @@ files:
 - lib/gnip_api/json_parser.rb
 - lib/gnip_api/power_track/buffer.rb
 - lib/gnip_api/power_track/rule.rb
-- lib/gnip_api/power_track/rule_validator.rb
 - lib/gnip_api/power_track/rules.rb
 - lib/gnip_api/power_track/stream.rb
 - lib/gnip_api/request.rb

data/lib/gnip_api/power_track/rule_validator.rb

@@ -1,6 +0,0 @@
-module GnipApi
-  module PowerTrack
-    class RuleValidator
-    end
-  end
-end