twitterscraper-ruby 0.11.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4382801b03a5384095aad6a955caea438787fa2eed96e3e001237df368925a2
-  data.tar.gz: 6722b4edce7242b3006e5c097dd78847f36e2da7edea009e2d7b89b09f5b25ff
+  metadata.gz: e98afb0444b724e0c9c29f6b888c017166859d1252337f34686526060ca8368d
+  data.tar.gz: 5ef3ff7f86d9a0c9dd1883d55498049d0f164aa7c71a7c9c2bbf0a89ae9bb32c
 SHA512:
-  metadata.gz: 4ca72a0bbce553c38061e0362f755a5e82b47a5288108508410c19a7eef9a2514b58682e88ed1bf89654d5b89c84c41edd8a5fa34fd7d1e5fbf92b267402884a
-  data.tar.gz: 8853b015cb37180d6814710d971a757d08aa4ddd4579af4131e204e34bb10c80ef3139c082f17be92303d9efc2e3f8eb4ba0d15bdf4f264fb4fba0cf87ed42d7
+  metadata.gz: 04ef61c57545cbbdbbe5da53d1f24cf064b7d1c61ad3da9bc57a361d24ed24480c4f68fa1fea67345ceff4d4d4685f046a4586f55ebe8f3dc0ca6332c7c2d928
+  data.tar.gz: f5fd19c8289c7caf574dc78f754ba9aaf9446f3819b394d14414909b1505e0f9b25181802448d28285de8db81a27e12e0e65d1e1a0b2b0e5df8e7e73d6263e14

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    twitterscraper-ruby (0.11.0)
+    twitterscraper-ruby (0.12.0)
       nokogiri
       parallel
 

data/README.md

@@ -137,8 +137,8 @@ $ cat tweets.json | jq . | less
 | ------------- | ------------- | ------------- |
 | `-h`, `--help` | This option displays a summary of twitterscraper. | |
 | `--query` | Specify a keyword used during the search. | |
-| `--start_date` | Set the date from which twitterscraper-ruby should start scraping for your query. | |
-| `--end_date` | Set the enddate which twitterscraper-ruby should use to stop scraping for your query. | |
+| `--start_date` | Used as "since:yyyy-mm-dd for your query. This means "since the date". | |
+| `--end_date` | Used as "until:yyyy-mm-dd for your query. This means "before the date". | |
 | `--lang` | Retrieve tweets written in a specific language. | |
 | `--limit` | Stop scraping when *at least* the number of tweets indicated with --limit is scraped. | 100 |
 | `--threads` | Set the number of threads twitterscraper-ruby should initiate while scraping for your query. | 2 |

data/lib/twitterscraper/cli.rb

@@ -20,6 +20,7 @@ module Twitterscraper
           end_date: options['end_date'],
           lang: options['lang'],
           limit: options['limit'],
+          daily_limit: options['daily_limit'],
           threads: options['threads'],
           proxy: options['proxy']
       }
@@ -63,6 +64,7 @@ module Twitterscraper
           'end_date:',
           'lang:',
           'limit:',
+          'daily_limit:',
           'threads:',
           'output:',
           'format:',
@@ -72,8 +74,10 @@ module Twitterscraper
           'verbose',
       )
 
+      options['start_date'] = Query::OLDEST_DATE if options['start_date'] == 'oldest'
       options['lang'] ||= ''
       options['limit'] = (options['limit'] || 100).to_i
+      options['daily_limit'] = options['daily_limit'].to_i if options['daily_limit']
       options['threads'] = (options['threads'] || 2).to_i
       options['format'] ||= 'json'
       options['output'] ||= "tweets.#{options['format']}"
@@ -101,7 +105,7 @@ module Twitterscraper
     end
 
     def print_version
-      puts "twitterscraper-#{Twitterscraper::VERSION}"
+      puts "twitterscraper-#{VERSION}"
     end
   end
 end

data/lib/twitterscraper/client.rb

@@ -2,7 +2,7 @@ module Twitterscraper
   class Client
     include Query
 
-    def initialize(cache:)
+    def initialize(cache: false)
       @cache = cache
     end
 

data/lib/twitterscraper/proxy.rb

@@ -17,15 +17,17 @@ module Twitterscraper
           reload
         end
         @cur_index += 1
-        item = @items[@cur_index - 1]
-        Twitterscraper.logger.info("Using proxy #{item}")
-        item
+        @items[@cur_index - 1]
       end
 
       def size
         @items.size
       end
 
+      def empty?
+        @items.empty?
+      end
+
       private
 
       def reload
@@ -51,7 +53,6 @@ module Twitterscraper
         proxies << ip + ':' + port
       end
 
-      Twitterscraper.logger.debug "Fetch #{proxies.size} proxies"
       proxies.shuffle
     rescue => e
       if (retries -= 1) > 0

data/lib/twitterscraper/query.rb

@@ -44,14 +44,18 @@ module Twitterscraper
 
     def get_single_page(url, headers, proxies, timeout = 6, retries = 30)
       return nil if stop_requested?
-      Twitterscraper::Http.get(url, headers, proxies.sample, timeout)
+      unless proxies.empty?
+        proxy = proxies.sample
+        logger.info("Using proxy #{proxy}")
+      end
+      Http.get(url, headers, proxy, timeout)
     rescue => e
       logger.debug "query_single_page: #{e.inspect}"
       if (retries -= 1) > 0
-        logger.info("Retrying... (Attempts left: #{retries - 1})")
+        logger.info "Retrying... (Attempts left: #{retries - 1})"
         retry
       else
-        raise
+        raise Error.new("#{e.inspect} url=#{url}")
       end
     end
 
@@ -71,27 +75,27 @@ module Twitterscraper
     end
 
     def query_single_page(query, lang, pos, from_user = false, headers: [], proxies: [])
-      logger.info("Querying #{query}")
+      logger.info "Querying #{query}"
       query = ERB::Util.url_encode(query)
 
       url = build_query_url(query, lang, pos, from_user)
       http_request = lambda do
-        logger.debug("Scraping tweets from #{url}")
+        logger.debug "Scraping tweets from #{url}"
         get_single_page(url, headers, proxies)
       end
 
       if cache_enabled?
         client = Cache.new
         if (response = client.read(url))
-          logger.debug('Fetching tweets from cache')
+          logger.debug 'Fetching tweets from cache'
         else
           response = http_request.call
-          client.write(url, response)
+          client.write(url, response) unless stop_requested?
         end
       else
         response = http_request.call
       end
-      return [], nil if response.nil?
+      return [], nil if response.nil? || response.empty?
 
       html, json_resp = parse_single_page(response, pos.nil?)
 
@@ -114,31 +118,31 @@ module Twitterscraper
 
     def validate_options!(query, start_date:, end_date:, lang:, limit:, threads:, proxy:)
       if query.nil? || query == ''
-        raise 'Please specify a search query.'
+        raise Error.new('Please specify a search query.')
       end
 
       if ERB::Util.url_encode(query).length >= 500
-        raise ':query must be a UTF-8, URL-encoded search query of 500 characters maximum, including operators.'
+        raise Error.new(':query must be a UTF-8, URL-encoded search query of 500 characters maximum, including operators.')
       end
 
       if start_date && end_date
         if start_date == end_date
-          raise 'Please specify different values for :start_date and :end_date.'
+          raise Error.new('Please specify different values for :start_date and :end_date.')
         elsif start_date > end_date
-          raise ':start_date must occur before :end_date.'
+          raise Error.new(':start_date must occur before :end_date.')
         end
       end
 
       if start_date
         if start_date < OLDEST_DATE
-          raise ":start_date must be greater than or equal to #{OLDEST_DATE}"
+          raise Error.new(":start_date must be greater than or equal to #{OLDEST_DATE}")
         end
       end
 
       if end_date
         today = Date.today
         if end_date > Date.today
-          raise ":end_date must be less than or equal to today(#{today})"
+          raise Error.new(":end_date must be less than or equal to today(#{today})")
         end
       end
     end
@@ -156,27 +160,32 @@ module Twitterscraper
       end
     end
 
-    def main_loop(query, lang, limit, headers, proxies)
+    def main_loop(query, lang, limit, daily_limit, headers, proxies)
       pos = nil
+      daily_tweets = []
 
       while true
         new_tweets, new_pos = query_single_page(query, lang, pos, headers: headers, proxies: proxies)
         unless new_tweets.empty?
+          daily_tweets.concat(new_tweets)
+          daily_tweets.uniq! { |t| t.tweet_id }
+
           @mutex.synchronize {
             @all_tweets.concat(new_tweets)
             @all_tweets.uniq! { |t| t.tweet_id }
           }
         end
-        logger.info("Got #{new_tweets.size} tweets (total #{@all_tweets.size})")
+        logger.info "Got #{new_tweets.size} tweets (total #{@all_tweets.size})"
 
         break unless new_pos
+        break if daily_limit && daily_tweets.size >= daily_limit
         break if @all_tweets.size >= limit
 
         pos = new_pos
       end
 
-      if @all_tweets.size >= limit
-        logger.info("Limit reached #{@all_tweets.size}")
+      if !@stop_requested && @all_tweets.size >= limit
+        logger.warn "The limit you specified has been reached limit=#{limit} tweets=#{@all_tweets.size}"
         @stop_requested = true
       end
     end
@@ -185,32 +194,36 @@ module Twitterscraper
       @stop_requested
     end
 
-    def query_tweets(query, start_date: nil, end_date: nil, lang: '', limit: 100, threads: 2, proxy: false)
+    def query_tweets(query, start_date: nil, end_date: nil, lang: '', limit: 100, daily_limit: nil, threads: 2, proxy: false)
       start_date = Date.parse(start_date) if start_date && start_date.is_a?(String)
       end_date = Date.parse(end_date) if end_date && end_date.is_a?(String)
       queries = build_queries(query, start_date, end_date)
       threads = queries.size if threads > queries.size
-      proxies = proxy ? Twitterscraper::Proxy::Pool.new : []
+      proxies = proxy ? Proxy::Pool.new : []
 
       validate_options!(queries[0], start_date: start_date, end_date: end_date, lang: lang, limit: limit, threads: threads, proxy: proxy)
 
-      logger.info("The number of threads #{threads}")
+      logger.debug "Fetch #{proxies.size} proxies" if proxy
+      logger.info "The number of threads #{threads}"
 
       headers = {'User-Agent': USER_AGENT_LIST.sample, 'X-Requested-With': 'XMLHttpRequest'}
-      logger.info("Headers #{headers}")
+      logger.info "Headers #{headers}"
 
       @all_tweets = []
       @mutex = Mutex.new
       @stop_requested = false
 
       if threads > 1
+        Thread.abort_on_exception = true
+        logger.debug "Set 'Thread.abort_on_exception' to true"
+
         Parallel.each(queries, in_threads: threads) do |query|
-          main_loop(query, lang, limit, headers, proxies)
+          main_loop(query, lang, limit, daily_limit, headers, proxies)
           raise Parallel::Break if stop_requested?
         end
       else
         queries.each do |query|
-          main_loop(query, lang, limit, headers, proxies)
+          main_loop(query, lang, limit, daily_limit, headers, proxies)
           break if stop_requested?
         end
       end

data/lib/twitterscraper/tweet.rb

@@ -59,12 +59,19 @@ module Twitterscraper
       def from_tweets_html(html)
         html.map do |tweet|
           from_tweet_html(tweet)
-        end
+        end.compact
       end
 
       def from_tweet_html(html)
+        screen_name = html.attr('data-screen-name')
+        tweet_id = html.attr('data-tweet-id')&.to_i
+
+        unless html.to_s.include?('js-tweet-text-container')
+          Twitterscraper.logger.warn "html doesn't include div.js-tweet-text-container url=https://twitter.com/#{screen_name}/status/#{tweet_id}"
+          return nil
+        end
+
         inner_html = Nokogiri::HTML(html.inner_html)
-        tweet_id = html.attr('data-tweet-id').to_i
         text = inner_html.xpath("//div[@class[contains(., 'js-tweet-text-container')]]/p[@class[contains(., 'js-tweet-text')]]").first.text
         links = inner_html.xpath("//a[@class[contains(., 'twitter-timeline-link')]]").map { |elem| elem.attr('data-expanded-url') }.select { |link| link && !link.include?('pic.twitter') }
         image_urls = inner_html.xpath("//div[@class[contains(., 'AdaptiveMedia-photoContainer')]]").map { |elem| elem.attr('data-image-url') }
@@ -89,7 +96,7 @@ module Twitterscraper
 
         timestamp = inner_html.xpath("//span[@class[contains(., 'js-short-timestamp')]]").first.attr('data-time').to_i
         new(
-            screen_name: html.attr('data-screen-name'),
+            screen_name: screen_name,
             name: html.attr('data-name'),
             user_id: html.attr('data-user-id').to_i,
             tweet_id: tweet_id,

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Twitterscraper
-  VERSION = '0.11.0'
+  VERSION = '0.12.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: twitterscraper-ruby
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.0
 platform: ruby
 authors:
 - ts-3156