twitterscraper-ruby 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eda9826c0c4afe8f4ee557a309d82330b0e970882e19d38d917d854ea4bd308b
-  data.tar.gz: 11b36f581640e7ab492b15364ed0521e7a15ad4d9b0e94d5b9d5aece36541d6a
+  metadata.gz: e6701ff59f3eb13db9e3b2d024ea983264b528194d64f4f03a95f3576338ed77
+  data.tar.gz: 74106816dd406ef1b355b4d4fc94b1baf4509465f6d5bf1ea8f7c654e518eec0
 SHA512:
-  metadata.gz: 990044f929c9dbcca4f17eb21730094cdc8d9aaf6b0a53eb012e55cd2738a26d3bd18dcc75456a8dd4d00a132faa1d32e4d04c2bcec5385ee1cfa554b4e7cfab
-  data.tar.gz: 6f50f5add0359866a2c4fa7f2ae78fb5dd96cbf3ab7525be847daee1a40015df2836b5900468159b34e24641cde7dc07267f53ee1e29ccea0401b5f85080f44b
+  metadata.gz: 9710fb74c90dcbc17a22dd613cfe4dce75106951f1e55cd9cfa94a825ecf0b6773a2851ff1cca842f83b5207d3744ac63bcce29031061b0a0ac84cc12d62b8a3
+  data.tar.gz: f0e7cd90ecb773a1837be9245b83f51d60f25d48cab716e3584a8d3e1b6f0fe4951eadaf034850599205209d2bf8d7cd4ebfda97f9f862a528c393a2f81887a7

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    twitterscraper-ruby (0.4.0)
+    twitterscraper-ruby (0.5.0)
       nokogiri
+      parallel
 
 GEM
   remote: https://rubygems.org/
@@ -11,6 +12,7 @@ GEM
     minitest (5.14.1)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
+    parallel (1.19.2)
     rake (12.3.3)
 
 PLATFORMS

data/lib/twitterscraper/cli.rb

@@ -13,7 +13,8 @@ module Twitterscraper
     def run
       client = Twitterscraper::Client.new
       limit = options['limit'] ? options['limit'].to_i : 100
-      tweets = client.query_tweets(options['query'], limit: limit, start_date: options['start_date'], end_date: options['end_date'])
+      threads = options['threads'] ? options['threads'].to_i : 2
+      tweets = client.query_tweets(options['query'], limit: limit, threads: threads, start_date: options['start_date'], end_date: options['end_date'])
       File.write('tweets.json', generate_json(tweets))
     end
 
@@ -36,6 +37,7 @@ module Twitterscraper
           'limit:',
           'start_date:',
           'end_date:',
+          'threads:',
           'pretty',
       )
     end

data/lib/twitterscraper/http.rb

@@ -9,7 +9,6 @@ module Twitterscraper
       if proxy
         ip, port = proxy.split(':')
         http_class = Net::HTTP::Proxy(ip, port.to_i)
-        Twitterscraper.logger.info("Using proxy #{proxy}")
       else
         http_class = Net::HTTP
       end

data/lib/twitterscraper/proxy.rb

@@ -8,7 +8,7 @@ module Twitterscraper
 
     class Result
       def initialize(items)
-        @items = items.shuffle
+        @items = items
         @cur_index = 0
       end
 
@@ -17,7 +17,9 @@ module Twitterscraper
           reload
         end
         @cur_index += 1
-        @items[@cur_index - 1]
+        item = @items[@cur_index - 1]
+        Twitterscraper.logger.info("Using proxy #{item}")
+        item
       end
 
       def size
@@ -27,7 +29,7 @@ module Twitterscraper
       private
 
       def reload
-        @items = Proxy.get_proxies.shuffle
+        @items = Proxy.get_proxies
         @cur_index = 0
         Twitterscraper.logger.debug "Reload #{proxies.size} proxies"
       end
@@ -50,7 +52,7 @@ module Twitterscraper
       end
 
       Twitterscraper.logger.debug "Fetch #{proxies.size} proxies"
-      Result.new(proxies)
+      Result.new(proxies.shuffle)
     rescue => e
       if (retries -= 1) > 0
         retry

data/lib/twitterscraper/query.rb

@@ -1,7 +1,9 @@
+require 'resolv-replace'
 require 'net/http'
 require 'nokogiri'
 require 'date'
 require 'json'
+require 'parallel'
 
 module Twitterscraper
   module Query
@@ -14,7 +16,6 @@ module Twitterscraper
         'Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16',
         'Mozilla/5.0 (Windows NT 5.2; RW; rv:7.0a1) Gecko/20091211 SeaMonkey/9.23a1pre',
     ]
-    USER_AGENT = USER_AGENT_LIST.sample
 
     INIT_URL = 'https://twitter.com/search?f=tweets&vertical=default&q=__QUERY__&l=__LANG__'
     RELOAD_URL = 'https://twitter.com/i/search/timeline?f=tweets&vertical=' +
@@ -59,15 +60,15 @@ module Twitterscraper
       else
         json_resp = JSON.parse(text)
         items_html = json_resp['items_html'] || ''
-        logger.debug json_resp['message'] if json_resp['message'] # Sorry, you are rate limited.
+        logger.warn json_resp['message'] if json_resp['message'] # Sorry, you are rate limited.
       end
 
       [items_html, json_resp]
     end
 
     def query_single_page(query, lang, pos, from_user = false, headers: [], proxies: [])
-      query = query.gsub(' ', '%20').gsub('#', '%23').gsub(':', '%3A').gsub('&', '%26')
       logger.info("Querying #{query}")
+      query = query.gsub(' ', '%20').gsub('#', '%23').gsub(':', '%3A').gsub('&', '%26')
 
       url = build_query_url(query, lang, pos, from_user)
       logger.debug("Scraping tweets from #{url}")
@@ -99,28 +100,31 @@ module Twitterscraper
         raise ':start_date must occur before :end_date.'
       end
 
-      # TODO parallel
-
-      pos = nil
-      all_tweets = []
-
       proxies = Twitterscraper::Proxy.get_proxies
 
-      headers = {'User-Agent': USER_AGENT, 'X-Requested-With': 'XMLHttpRequest'}
-      logger.info("Headers #{headers}")
+      date_range = start_date.upto(end_date - 1)
+      queries = date_range.map { |date| query + " since:#{date} until:#{date + 1}" }
+      threads = queries.size if threads > queries.size
+      logger.info("Threads #{threads}")
+
+      all_tweets = []
+      mutex = Mutex.new
 
-      start_date.upto(end_date) do |date|
-        break if date == end_date
+      Parallel.each(queries, in_threads: threads) do |query|
+        headers = {'User-Agent': USER_AGENT_LIST.sample, 'X-Requested-With': 'XMLHttpRequest'}
+        logger.info("Headers #{headers}")
 
-        queries = query + " since:#{date} until:#{date + 1}"
+        pos = nil
 
         while true
-          new_tweets, new_pos = query_single_page(queries, lang, pos, headers: headers, proxies: proxies)
+          new_tweets, new_pos = query_single_page(query, lang, pos, headers: headers, proxies: proxies)
           unless new_tweets.empty?
-            all_tweets.concat(new_tweets)
-            all_tweets.uniq! { |t| t.tweet_id }
+            mutex.synchronize {
+              all_tweets.concat(new_tweets)
+              all_tweets.uniq! { |t| t.tweet_id }
+            }
           end
-          logger.info("Got #{new_tweets.size} tweets (total #{all_tweets.size})")
+          logger.info("Got #{new_tweets.size} tweets (total #{all_tweets.size}) worker=#{Parallel.worker_number}")
 
           break unless new_pos
           break if all_tweets.size >= limit
@@ -130,11 +134,11 @@ module Twitterscraper
 
         if all_tweets.size >= limit
           logger.info("Reached limit #{all_tweets.size}")
-          break
+          raise Parallel::Break
         end
       end
 
-      all_tweets
+      all_tweets.sort_by { |tweet| -tweet.created_at.to_i }
     end
   end
 end

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Twitterscraper
-  VERSION = "0.4.0"
+  VERSION = '0.5.0'
 end

data/twitterscraper-ruby.gemspec

@@ -27,4 +27,5 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = ">= 2.6.4"
 
   spec.add_dependency "nokogiri"
+  spec.add_dependency "parallel"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twitterscraper-ruby
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - ts-3156
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-12 00:00:00.000000000 Z
+date: 2020-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A gem  to scrape Tweets
 email:
 - ts_3156@yahoo.co.jp