twitterscraper-ruby 0.18.0 → 0.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e9bdefe1c4d10e6d9f1d12aeb279b2a3751c570e96e05daaf849dd423bb03bf
-  data.tar.gz: 7de97de19daeecce2837fe8e5999b6c9490ab49a18a2ab9e603bf4d039abc4b9
+  metadata.gz: 2056b4a3d9fe7af49429e35b3a1688256fb31b74cabab841a4dd2376a79889d5
+  data.tar.gz: aaaf949da2ba2ae07a0d66e981aebc635c18120de06be705f96c19c92c309911
 SHA512:
-  metadata.gz: 55b7e0b52b2ce44418305798ed27a677405244a48f5ad0a797e3abf7958b0581a313ebd33f3f69b891ba7454f8f5c9c0db845c9ca8be321cd27212932821776e
-  data.tar.gz: 8fe97a0dc164fc0108b8e6a35843fba19ade5fbaf4f1ee2b4a400afbd3bdbb220a49dfbef4fceb1d8ecc43df3b4f4b7bad0ee5ea94c0aac464c0477e42efb866
+  metadata.gz: c60824e4c1c0021a3e27451b1708a77bd2e15dd6258fce63ac1b95111d0230c8ab7317bcd76c2faf14d02ebe75ab8d7453924e01eee7d3fcb46eef374f16c575
+  data.tar.gz: 984204bd430b41b76a2d9108df4e778e2bb242010ebd18569bcb662473496826644ba5693db1d475d565bff49a3de7f0eb95fd4c9a3da9e5ed4d6a6219ebb62e

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    twitterscraper-ruby (0.18.0)
+    twitterscraper-ruby (0.19.0)
       nokogiri
       parallel
 

data/lib/twitterscraper/cache.rb

@@ -4,7 +4,7 @@ require 'digest/md5'
 module Twitterscraper
   class Cache
     def initialize()
-      @ttl = 86400 # 1 day
+      @ttl = 86400 * 3 # 3 day
       @dir = 'cache'
       Dir.mkdir(@dir) unless File.exist?(@dir)
     end
@@ -25,6 +25,12 @@ module Twitterscraper
       File.write(file, entry.to_json)
     end
 
+    def exist?(key)
+      key = cache_key(key)
+      file = File.join(@dir, key)
+      File.exist?(file)
+    end
+
     def delete(key)
       key = cache_key(key)
       file = File.join(@dir, key)

data/lib/twitterscraper/cli.rb

@@ -32,16 +32,17 @@ module Twitterscraper
     end
 
     def export(name, tweets)
-      filepath = options['output']
-      Dir.mkdir(File.dirname(filepath)) unless File.exist?(File.dirname(filepath))
-      write_json = lambda { File.write(filepath, generate_json(tweets)) }
-
-      if options['format'] == 'json'
-        write_json.call
-      elsif options['format'] == 'html'
-        File.write(filepath, Template.new.tweets_embedded_html(name, tweets, options))
-      else
-        write_json.call
+      options['format'].split(',').map(&:strip).each do |format|
+        file = build_output_name(format, options)
+        Dir.mkdir(File.dirname(file)) unless File.exist?(File.dirname(file))
+
+        if format == 'json'
+          File.write(file, generate_json(tweets))
+        elsif format == 'html'
+          File.write(file, Template.new.tweets_embedded_html(name, tweets, options))
+        else
+          puts "Invalid format #{format}"
+        end
       end
     end
 
@@ -90,7 +91,6 @@ module Twitterscraper
       options['threads_granularity'] ||= 'auto'
       options['format'] ||= 'json'
       options['order'] ||= 'desc'
-      options['output'] ||= build_output_name(options)
 
       options['cache'] = options['cache'] != 'false'
       options['proxy'] = options['proxy'] != 'false'
@@ -98,10 +98,11 @@ module Twitterscraper
       options
     end
 
-    def build_output_name(options)
+    def build_output_name(format, options)
       query = options['query'].gsub(/[ :?#&]/, '_')
       date = [options['start_date'], options['end_date']].select { |val| val && !val.empty? }.join('_')
-      File.join('out', [options['type'], 'tweets', date, query].compact.join('_') + '.' + options['format'])
+      file = [options['type'], 'tweets', date, query].compact.join('_') + '.' + format
+      File.join('out', file)
     end
 
     def initialize_logger

data/lib/twitterscraper/client.rb

@@ -2,9 +2,31 @@ module Twitterscraper
   class Client
     include Query
 
+    USER_AGENT_LIST = [
+        'Mozilla/5.0 (Windows; U; Windows NT 6.1; x64; fr; rv:1.9.2.13) Gecko/20101203 Firebird/3.6.13',
+        'Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko',
+        'Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:2.2) Gecko/20110201',
+        'Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16',
+        'Mozilla/5.0 (Windows NT 5.2; RW; rv:7.0a1) Gecko/20091211 SeaMonkey/9.23a1pre',
+    ]
+
     def initialize(cache: true, proxy: true)
+      @request_headers = {'User-Agent': USER_AGENT_LIST.sample, 'X-Requested-With': 'XMLHttpRequest'}
+      Twitterscraper.logger.info "Headers #{@request_headers}"
+
       @cache = cache
-      @proxy = proxy
+
+      if (@proxy = proxy)
+        @proxies = Proxy::Pool.new
+        Twitterscraper.logger.debug "Fetch #{@proxies.size} proxies"
+      else
+        @proxies = []
+        Twitterscraper.logger.debug 'Proxy disabled'
+      end
+    end
+
+    def request_headers
+      @request_headers
     end
 
     def cache_enabled?
@@ -14,5 +36,9 @@ module Twitterscraper
     def proxy_enabled?
       @proxy
     end
+
+    def proxies
+      @proxies
+    end
   end
 end

data/lib/twitterscraper/query.rb

@@ -10,14 +10,6 @@ module Twitterscraper
   module Query
     include Logger
 
-    USER_AGENT_LIST = [
-        'Mozilla/5.0 (Windows; U; Windows NT 6.1; x64; fr; rv:1.9.2.13) Gecko/20101203 Firebird/3.6.13',
-        'Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko',
-        'Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:2.2) Gecko/20110201',
-        'Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16',
-        'Mozilla/5.0 (Windows NT 5.2; RW; rv:7.0a1) Gecko/20091211 SeaMonkey/9.23a1pre',
-    ]
-
     INIT_URL = 'https://twitter.com/search?f=tweets&vertical=default&q=__QUERY__&l=__LANG__'
     RELOAD_URL = 'https://twitter.com/i/search/timeline?f=tweets&vertical=' +
         'default&include_available_features=1&include_entities=1&' +
@@ -43,13 +35,13 @@ module Twitterscraper
       end
     end
 
-    def get_single_page(url, headers, proxies, timeout = 6, retries = 30)
+    def get_single_page(url, timeout = 6, retries = 30)
       return nil if stop_requested?
-      unless proxies.empty?
+      if proxy_enabled?
         proxy = proxies.sample
         logger.info("Using proxy #{proxy}")
       end
-      Http.get(url, headers, proxy, timeout)
+      Http.get(url, request_headers, proxy, timeout)
     rescue => e
       logger.debug "get_single_page: #{e.inspect}"
       if (retries -= 1) > 0
@@ -74,14 +66,14 @@ module Twitterscraper
       [items_html, json_resp]
     end
 
-    def query_single_page(query, lang, type, pos, headers: [], proxies: [])
+    def query_single_page(query, lang, type, pos)
       logger.info "Querying #{query}"
       encoded_query = ERB::Util.url_encode(query)
 
       url = build_query_url(encoded_query, lang, type, pos)
       http_request = lambda do
         logger.debug "Scraping tweets from url=#{url}"
-        get_single_page(url, headers, proxies)
+        get_single_page(url)
       end
 
       if cache_enabled?
@@ -160,7 +152,7 @@ module Twitterscraper
         if threads_granularity == 'day'
           date_range = start_date.upto(end_date - 1)
           queries = date_range.map { |date| query + " since:#{date} until:#{date + 1}" }
-        else
+        elsif threads_granularity == 'hour'
           time = Time.utc(start_date.year, start_date.month, start_date.day, 0, 0, 0)
           end_time = Time.utc(end_date.year, end_date.month, end_date.day, 0, 0, 0)
           queries = []
@@ -185,34 +177,35 @@ module Twitterscraper
       end
     end
 
-    def main_loop(query, lang, type, limit, daily_limit, headers, proxies)
+    def main_loop(query, lang, type, limit, daily_limit)
       pos = nil
-      daily_tweets = []
+      tmp_tweets = []
 
       while true
-        new_tweets, new_pos = query_single_page(query, lang, type, pos, headers: headers, proxies: proxies)
+        new_tweets, new_pos = query_single_page(query, lang, type, pos)
         unless new_tweets.empty?
-          daily_tweets.concat(new_tweets)
-          daily_tweets.uniq! { |t| t.tweet_id }
+          tmp_tweets.concat(new_tweets)
+          tmp_tweets.uniq! { |t| t.tweet_id }
+        end
 
-          @mutex.synchronize {
-            @all_tweets.concat(new_tweets)
-            @all_tweets.uniq! { |t| t.tweet_id }
-          }
+        @results_counter[Parallel.worker_number] = tmp_tweets.size
+        total_size = @all_tweets.size + @results_counter.values.sum
+        logger.info "Got tweets new=#{new_tweets.size} tmp=#{tmp_tweets.size} all=#{@all_tweets.size} total=#{total_size}"
+
+        if !@stop_requested && total_size >= limit
+          logger.warn "The limit you specified has been reached limit=#{limit} tweets=#{total_size}"
+          @stop_requested = true
         end
-        logger.info "Got #{new_tweets.size} tweets (total #{@all_tweets.size})"
 
         break unless new_pos
-        break if daily_limit && daily_tweets.size >= daily_limit
+        break if @stop_requested
+        break if daily_limit && tmp_tweets.size >= daily_limit
         break if @all_tweets.size >= limit
 
         pos = new_pos
       end
 
-      if !@stop_requested && @all_tweets.size >= limit
-        logger.warn "The limit you specified has been reached limit=#{limit} tweets=#{@all_tweets.size}"
-        @stop_requested = true
-      end
+      tmp_tweets
     end
 
     def stop_requested?
@@ -233,13 +226,6 @@ module Twitterscraper
       if threads > queries.size
         threads = queries.size
       end
-      if proxy_enabled?
-        proxies = Proxy::Pool.new
-        logger.debug "Fetch #{proxies.size} proxies"
-      else
-        proxies = []
-        logger.debug 'Proxy disabled'
-      end
       logger.debug "Cache #{cache_enabled? ? 'enabled' : 'disabled'}"
 
       validate_options!(queries, type: type, start_date: start_date, end_date: end_date, lang: lang, limit: limit, threads: threads)
@@ -247,24 +233,32 @@ module Twitterscraper
       logger.info "The number of queries #{queries.size}"
       logger.info "The number of threads #{threads}"
 
-      headers = {'User-Agent': USER_AGENT_LIST.sample, 'X-Requested-With': 'XMLHttpRequest'}
-      logger.info "Headers #{headers}"
-
       @all_tweets = []
-      @mutex = Mutex.new
       @stop_requested = false
+      @results_counter = {}
 
       if threads > 1
+        @mutex = Mutex.new
         Thread.abort_on_exception = true
         logger.debug "Set 'Thread.abort_on_exception' to true"
 
         Parallel.each(queries, in_threads: threads) do |query|
-          main_loop(query, lang, type, limit, daily_limit, headers, proxies)
+          @results_counter[Parallel.worker_number] = 0
+          tmp_tweets = main_loop(query, lang, type, limit, daily_limit)
+          @mutex.synchronize {
+            @all_tweets.concat(tmp_tweets)
+            @all_tweets.uniq! { |t| t.tweet_id }
+          }
+          @results_counter[Parallel.worker_number] = 0
+
           raise Parallel::Break if stop_requested?
         end
       else
         queries.each do |query|
-          main_loop(query, lang, type, limit, daily_limit, headers, proxies)
+          tmp_tweets = main_loop(query, lang, type, limit, daily_limit)
+          @all_tweets.concat(tmp_tweets)
+          @all_tweets.uniq! { |t| t.tweet_id }
+
           break if stop_requested?
         end
       end

data/lib/twitterscraper/template.rb

@@ -16,39 +16,37 @@ module Twitterscraper
       )
     end
 
-    def chart_data(tweets, trimming: true, smoothing: true)
-      min_interval = 5
-
-      data = tweets.each_with_object(Hash.new(0)) do |tweet, memo|
-        t = tweet.created_at
-        min = (t.min.to_f / min_interval).floor * min_interval
-        time = Time.new(t.year, t.month, t.day, t.hour, min, 0, '+00:00')
-        memo[time.to_i] += 1
-      end
-
-      if false && trimming
-        data.keys.sort.each.with_index do |timestamp, i|
-          break if data.size - 1 == i
-          if data[i] == 0 && data[i + 1] == 0
-            data.delete(timestamp)
+    def chart_data(tweets, grouping: 'auto')
+      if grouping && tweets.size > 100
+        if grouping == 'auto'
+          month = 28 * 24 * 60 * 60 # 28 days
+          duration = tweets[-1].created_at - tweets[0].created_at
+
+          if duration > 3 * month
+            grouping = 'day'
+          elsif duration > month || tweets.size > 10000
+            grouping = 'hour'
+          else
+            grouping = 'minute'
           end
         end
       end
 
-      if false && smoothing
-        time = data.keys.min
-        max_time = data.keys.max
-        sec_interval = 60 * min_interval
+      Twitterscraper.logger.info "Chart grouping #{grouping}"
 
-        while true
-          next_time = time + sec_interval
-          break if next_time + sec_interval > max_time
+      data = tweets.each_with_object(Hash.new(0)) do |tweet, memo|
+        t = tweet.created_at
 
-          unless data.has_key?(next_time)
-            data[next_time] = (data[time] + data[next_time + sec_interval]) / 2
-          end
-          time = next_time
+        if grouping == 'day'
+          time = Time.new(t.year, t.month, t.day, 0, 0, 0, '+00:00')
+        elsif grouping == 'hour'
+          time = Time.new(t.year, t.month, t.day, t.hour, 0, 0, '+00:00')
+        elsif grouping == 'minute'
+          time = Time.new(t.year, t.month, t.day, t.hour, t.min, 0, '+00:00')
+        else
+          time = t
         end
+        memo[time.to_i] += 1
       end
 
       data.sort_by { |k, _| k }.map do |timestamp, count|

data/lib/twitterscraper/template/tweets.html.erb

@@ -1,5 +1,8 @@
-<html>
+<!DOCTYPE html>
+<html lang="ja">
 <head>
+  <meta charset="UTF-8">
+
   <script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.27.0/moment.min.js" integrity="sha512-rmZcZsyhe0/MAjquhTgiUcb4d9knaFc7b5xAfju483gbEXTkeJRUMIPk6s3ySZMYUHEcjKbjLjyddGWMrNEvZg==" crossorigin="anonymous"></script>
   <script src="https://cdnjs.cloudflare.com/ajax/libs/moment-timezone/0.5.31/moment-timezone-with-data.min.js" integrity="sha512-HZcf3uHWA+Y2P5KNv+F/xa87/flKVP92kUTe/KXjU8URPshczF1Dx+cL5bw0VBGhmqWAK0UbhcqxBbyiNtAnWQ==" crossorigin="anonymous"></script>
   <script src="https://code.highcharts.com/stock/highstock.js"></script>
@@ -80,7 +83,7 @@
 <div id="chart-container"><div style="color: gray;">Loading...</div></div>
 
 <div class="tweets-container">
-  <% tweets.sort_by { |t| -t.created_at.to_i }.each.with_index do |tweet, i| %>
+  <% tweets.sort_by { |t| -t.created_at.to_i }.take(1000).each.with_index do |tweet, i| %>
     <% tweet_time = tweet.created_at.localtime.strftime('%Y-%m-%d %H:%M') %>
     <% if i < convert_limit %>
       <blockquote class="twitter-tweet">

data/lib/twitterscraper/type.rb

@@ -11,5 +11,9 @@ module Twitterscraper
     def user?
       @value == 'user'
     end
+
+    def to_s
+      @value
+    end
   end
 end

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Twitterscraper
-  VERSION = '0.18.0'
+  VERSION = '0.19.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twitterscraper-ruby
 version: !ruby/object:Gem::Version
-  version: 0.18.0
+  version: 0.19.0
 platform: ruby
 authors:
 - ts-3156
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-19 00:00:00.000000000 Z
+date: 2020-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri