twitterscraper-ruby 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 68f4f63474e0c165316575e8d5fc2ac8ec9cdb18218be2f084263effbdf78bb0
-  data.tar.gz: d91edfdbd1cc36f4bf722e5a4673238a814846066370cd6810af483d089d1768
+  metadata.gz: eda9826c0c4afe8f4ee557a309d82330b0e970882e19d38d917d854ea4bd308b
+  data.tar.gz: 11b36f581640e7ab492b15364ed0521e7a15ad4d9b0e94d5b9d5aece36541d6a
 SHA512:
-  metadata.gz: a7b0f2ce114a2eef72be3147d9459f7b91f15813825cd65baf10b0c891a64a9911c6b4e15db72425925e287043e46ac73cc956f088eb03c5ad960d213b4b4175
-  data.tar.gz: 8f42a24221aebc9fa361b7dbf9bb23cc683112b03bd549456fdd6f49bf8763d5d8797c73ce1d5cfe60d534f329d649d845a741baedcefaca46a17dd194055778
+  metadata.gz: 990044f929c9dbcca4f17eb21730094cdc8d9aaf6b0a53eb012e55cd2738a26d3bd18dcc75456a8dd4d00a132faa1d32e4d04c2bcec5385ee1cfa554b4e7cfab
+  data.tar.gz: 6f50f5add0359866a2c4fa7f2ae78fb5dd96cbf3ab7525be847daee1a40015df2836b5900468159b34e24641cde7dc07267f53ee1e29ccea0401b5f85080f44b

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    twitterscraper-ruby (0.3.0)
+    twitterscraper-ruby (0.4.0)
       nokogiri
 
 GEM

data/lib/twitterscraper/cli.rb

@@ -14,13 +14,21 @@ module Twitterscraper
       client = Twitterscraper::Client.new
       limit = options['limit'] ? options['limit'].to_i : 100
       tweets = client.query_tweets(options['query'], limit: limit, start_date: options['start_date'], end_date: options['end_date'])
-      File.write('tweets.json', ::JSON.dump(tweets))
+      File.write('tweets.json', generate_json(tweets))
     end
 
     def options
       @options
     end
 
+    def generate_json(tweets)
+      if options['pretty']
+        ::JSON.pretty_generate(tweets)
+      else
+        ::JSON.generate(tweets)
+      end
+    end
+
     def parse_options(argv)
       argv.getopts(
           'h',
@@ -28,6 +36,7 @@ module Twitterscraper
           'limit:',
           'start_date:',
           'end_date:',
+          'pretty',
       )
     end
   end

data/lib/twitterscraper/http.rb

@@ -9,6 +9,7 @@ module Twitterscraper
       if proxy
         ip, port = proxy.split(':')
         http_class = Net::HTTP::Proxy(ip, port.to_i)
+        Twitterscraper.logger.info("Using proxy #{proxy}")
       else
         http_class = Net::HTTP
       end
@@ -24,7 +25,8 @@ module Twitterscraper
         req[key] = value
       end
 
-      http.request(req).body
+      res = http.start { http.request(req) }
+      res.body
     end
   end
 end

data/lib/twitterscraper/proxy.rb

@@ -6,22 +6,51 @@ module Twitterscraper
     class RetryExhausted < StandardError
     end
 
+    class Result
+      def initialize(items)
+        @items = items.shuffle
+        @cur_index = 0
+      end
+
+      def sample
+        if @cur_index >= @items.size
+          reload
+        end
+        @cur_index += 1
+        @items[@cur_index - 1]
+      end
+
+      def size
+        @items.size
+      end
+
+      private
+
+      def reload
+        @items = Proxy.get_proxies.shuffle
+        @cur_index = 0
+        Twitterscraper.logger.debug "Reload #{proxies.size} proxies"
+      end
+    end
+
     module_function
 
     def get_proxies(retries = 3)
       response = Twitterscraper::Http.get(PROXY_URL)
       html = Nokogiri::HTML(response)
-      table = html.xpath('//*[@id="proxylisttable"]').first
+      table = html.xpath('//table[@id="proxylisttable"]').first
 
       proxies = []
 
       table.xpath('tbody/tr').each do |tr|
         cells = tr.xpath('td')
-        ip, port = cells[0].text.strip, cells[1].text.strip
+        ip, port, https = [0, 1, 6].map { |i| cells[i].text.strip }
+        next if https == 'no'
         proxies << ip + ':' + port
       end
 
-      proxies
+      Twitterscraper.logger.debug "Fetch #{proxies.size} proxies"
+      Result.new(proxies)
     rescue => e
       if (retries -= 1) > 0
         retry

data/lib/twitterscraper/query.rb

@@ -25,7 +25,7 @@ module Twitterscraper
         'include_available_features=1&include_entities=1&' +
         'max_position={pos}&reset_error_state=false'
 
-    def get_query_url(query, lang, pos, from_user = false)
+    def build_query_url(query, lang, pos, from_user = false)
       # if from_user
       #   if !pos
       #     INIT_URL_USER.format(u = query)
@@ -40,52 +40,45 @@ module Twitterscraper
       end
     end
 
-    def query_single_page(query, lang, pos, retries = 30, from_user = false, timeout = 3, headers: [], proxies: [])
+    def get_single_page(url, headers, proxies, timeout = 10, retries = 30)
+      Twitterscraper::Http.get(url, headers, proxies.sample, timeout)
+    rescue => e
+      logger.debug "query_single_page: #{e.inspect}"
+      if (retries -= 1) > 0
+        logger.info("Retrying... (Attempts left: #{retries - 1})")
+        retry
+      else
+        raise
+      end
+    end
+
+    def parse_single_page(text, html = true)
+      if html
+        json_resp = nil
+        items_html = text
+      else
+        json_resp = JSON.parse(text)
+        items_html = json_resp['items_html'] || ''
+        logger.debug json_resp['message'] if json_resp['message'] # Sorry, you are rate limited.
+      end
+
+      [items_html, json_resp]
+    end
+
+    def query_single_page(query, lang, pos, from_user = false, headers: [], proxies: [])
       query = query.gsub(' ', '%20').gsub('#', '%23').gsub(':', '%3A').gsub('&', '%26')
       logger.info("Querying #{query}")
 
-      url = get_query_url(query, lang, pos, from_user)
+      url = build_query_url(query, lang, pos, from_user)
       logger.debug("Scraping tweets from #{url}")
 
-      response = nil
-      begin
-        proxy = proxies.sample
-        logger.info("Using proxy #{proxy}")
-
-        response = Twitterscraper::Http.get(url, headers, proxy, timeout)
-      rescue => e
-        logger.debug "query_single_page: #{e.inspect}"
-        if (retries -= 1) > 0
-          logger.info("Retrying... (Attempts left: #{retries - 1})")
-          retry
-        else
-          raise
-        end
-      end
-
-      html = ''
-      json_resp = nil
-
-      if pos
-        begin
-          json_resp = JSON.parse(response)
-          html = json_resp['items_html'] || ''
-        rescue => e
-          logger.warn("Failed to parse JSON #{e.inspect} while requesting #{url}")
-        end
-      else
-        html = response || ''
-      end
+      response = get_single_page(url, headers, proxies)
+      html, json_resp = parse_single_page(response, pos.nil?)
 
       tweets = Tweet.from_html(html)
 
       if tweets.empty?
-        if json_resp && json_resp['has_more_items']
-          pos = json_resp['min_position']
-        else
-          pos = nil
-        end
-        return [], pos
+        return [], (json_resp && json_resp['has_more_items'] && json_resp['min_position'])
       end
 
       if json_resp
@@ -103,7 +96,7 @@ module Twitterscraper
       if start_date == end_date
         raise 'Please specify different values for :start_date and :end_date.'
       elsif start_date > end_date
-        raise 'The :start_date must occur before :end_date.'
+        raise ':start_date must occur before :end_date.'
       end
 
       # TODO parallel
@@ -112,7 +105,6 @@ module Twitterscraper
       all_tweets = []
 
       proxies = Twitterscraper::Proxy.get_proxies
-      logger.info "Using #{proxies.size} proxies"
 
       headers = {'User-Agent': USER_AGENT, 'X-Requested-With': 'XMLHttpRequest'}
       logger.info("Headers #{headers}")
@@ -124,13 +116,11 @@ module Twitterscraper
 
         while true
           new_tweets, new_pos = query_single_page(queries, lang, pos, headers: headers, proxies: proxies)
-          logger.info("Got #{new_tweets.size} tweets")
-          logger.debug("new_pos=#{new_pos}")
-
           unless new_tweets.empty?
             all_tweets.concat(new_tweets)
             all_tweets.uniq! { |t| t.tweet_id }
           end
+          logger.info("Got #{new_tweets.size} tweets (total #{all_tweets.size})")
 
           break unless new_pos
           break if all_tweets.size >= limit
@@ -138,7 +128,10 @@ module Twitterscraper
           pos = new_pos
         end
 
-        break if all_tweets.size >= limit
+        if all_tweets.size >= limit
+          logger.info("Reached limit #{all_tweets.size}")
+          break
+        end
       end
 
       all_tweets

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Twitterscraper
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: twitterscraper-ruby
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - ts-3156