twitterscraper-ruby 0.16.0 → 0.20.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 66dda5275a9067d328f6637f127895ded954534d304e5e4b349f286a271a08d8
-  data.tar.gz: 6c3ffb3fba82376fc2de49514245ea96c7cb4fa16c32dcd2fff1ab1ae327bd14
+  metadata.gz: 8cb289da12a175a02664132b076349edf457585141b4bd196f1e2fb78ea69587
+  data.tar.gz: e9b3c55ee7e096b26746473d0b0dd12b7e259331daa6cae304571f902de72dd9
 SHA512:
-  metadata.gz: 24267284f4f29adc86d5bbe70a30bbe31d6d898546576065f1a9accafc3944a352117bbf6eb0de273743a00fb2d26c5cf37ed016cc0324187a25ca279230d812
-  data.tar.gz: 0bc9f01659560c83b0289bf63119849135b7ec27520dd03c7abd645da99ef660ca4b5fd12301b359cd5cc45a82914d7ceae88ad93ad756fde166718b3d0fe6c2
+  metadata.gz: 49a0d32d438c6c202257b733a877624429771cd6a57d3981716df5e0d946fc4b1af87f18d9029d5b43fb9df65a2a4a06579851d979514302999b85612d01f3e5
+  data.tar.gz: a59d26670417db7c57d203486b04798510a5f1c85468dde63aa1e7875c4151bc56169a7090ad745b14a08d473e8882a24e75b9ab0d04ae4bcece5912c8c2a3a5

data/.gitignore

@@ -8,3 +8,4 @@
 /tmp/
 /cache
 /.idea
+.DS_Store

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    twitterscraper-ruby (0.16.0)
+    twitterscraper-ruby (0.20.1)
       nokogiri
       parallel
 

data/README.md

@@ -98,6 +98,7 @@ end
       "screen_name": "@name",
       "name": "Name",
       "user_id": 12340000,
+      "profile_image_url": "https://pbs.twimg.com/profile_images/1826000000/0000.png",
       "tweet_id": 1234000000000000,
       "text": "Thanks Twitter!",
       "links": [],
@@ -122,6 +123,7 @@ end
 - screen_name
 - name
 - user_id
+- profile_image_url
 - tweet_id
 - text
 - links
@@ -173,6 +175,8 @@ Search operators documentation is in [Standard search operators](https://develop
 | `--limit`      | integer | Stop scraping when *at least* the number of tweets indicated with --limit is scraped. | 100 |
 | `--order`      | string  | Sort a order of the results. | desc(default) or asc |
 | `--threads`    | integer | Set the number of threads twitterscraper-ruby should initiate while scraping for your query. | 2 |
+| `--threads_granularity` | string | day or hour | auto |
+| `--chart_grouping` | string | day, hour or minute | auto |
 | `--proxy`      | boolean | Scrape https://twitter.com/search via proxies. | true(default) or false |
 | `--cache`      | boolean | Enable caching. | true(default) or false |
 | `--format`     | string  | The format of the output. | json(default) or html |

data/lib/twitterscraper/cache.rb

@@ -4,7 +4,7 @@ require 'digest/md5'
 module Twitterscraper
   class Cache
     def initialize()
-      @ttl = 86400 # 1 day
+      @ttl = 86400 * 3 # 3 day
       @dir = 'cache'
       Dir.mkdir(@dir) unless File.exist?(@dir)
     end
@@ -25,6 +25,12 @@ module Twitterscraper
       File.write(file, entry.to_json)
     end
 
+    def exist?(key)
+      key = cache_key(key)
+      file = File.join(@dir, key)
+      File.exist?(file)
+    end
+
     def delete(key)
       key = cache_key(key)
       file = File.join(@dir, key)

data/lib/twitterscraper/cli.rb

@@ -24,6 +24,7 @@ module Twitterscraper
           daily_limit: options['daily_limit'],
           order: options['order'],
           threads: options['threads'],
+          threads_granularity: options['threads_granularity'],
       }
       client = Twitterscraper::Client.new(cache: options['cache'], proxy: options['proxy'])
       tweets = client.query_tweets(options['query'], query_options)
@@ -31,14 +32,17 @@ module Twitterscraper
     end
 
     def export(name, tweets)
-      write_json = lambda { File.write(options['output'], generate_json(tweets)) }
-
-      if options['format'] == 'json'
-        write_json.call
-      elsif options['format'] == 'html'
-        File.write(options['output'], Template.new.tweets_embedded_html(name, tweets, options))
-      else
-        write_json.call
+      options['format'].split(',').map(&:strip).each do |format|
+        file = build_output_name(format, options)
+        Dir.mkdir(File.dirname(file)) unless File.exist?(File.dirname(file))
+
+        if format == 'json'
+          File.write(file, generate_json(tweets))
+        elsif format == 'html'
+          File.write(file, Template.new.tweets_embedded_html(name, tweets, options))
+        else
+          puts "Invalid format #{format}"
+        end
       end
     end
 
@@ -69,6 +73,8 @@ module Twitterscraper
           'daily_limit:',
           'order:',
           'threads:',
+          'threads_granularity:',
+          'chart_grouping:',
           'output:',
           'format:',
           'cache:',
@@ -82,10 +88,10 @@ module Twitterscraper
       options['lang'] ||= ''
       options['limit'] = (options['limit'] || 100).to_i
       options['daily_limit'] = options['daily_limit'].to_i if options['daily_limit']
-      options['threads'] = (options['threads'] || 2).to_i
+      options['threads'] = (options['threads'] || 10).to_i
+      options['threads_granularity'] ||= 'auto'
       options['format'] ||= 'json'
       options['order'] ||= 'desc'
-      options['output'] ||= "tweets.#{options['format']}"
 
       options['cache'] = options['cache'] != 'false'
       options['proxy'] = options['proxy'] != 'false'
@@ -93,6 +99,13 @@ module Twitterscraper
       options
     end
 
+    def build_output_name(format, options)
+      query = options['query'].gsub(/[ :?#&]/, '_')
+      date = [options['start_date'], options['end_date']].select { |val| val && !val.empty? }.join('_')
+      file = [options['type'], 'tweets', date, query].compact.join('_') + '.' + format
+      File.join('out', file)
+    end
+
     def initialize_logger
       Twitterscraper.logger.level = ::Logger::DEBUG if options['verbose']
     end

data/lib/twitterscraper/client.rb

@@ -2,9 +2,31 @@ module Twitterscraper
   class Client
     include Query
 
+    USER_AGENT_LIST = [
+        'Mozilla/5.0 (Windows; U; Windows NT 6.1; x64; fr; rv:1.9.2.13) Gecko/20101203 Firebird/3.6.13',
+        'Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko',
+        'Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:2.2) Gecko/20110201',
+        'Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16',
+        'Mozilla/5.0 (Windows NT 5.2; RW; rv:7.0a1) Gecko/20091211 SeaMonkey/9.23a1pre',
+    ]
+
     def initialize(cache: true, proxy: true)
+      @request_headers = {'User-Agent': USER_AGENT_LIST.sample, 'X-Requested-With': 'XMLHttpRequest'}
+      Twitterscraper.logger.info "Headers #{@request_headers}"
+
       @cache = cache
-      @proxy = proxy
+
+      if (@proxy = proxy)
+        @proxies = Proxy::Pool.new
+        Twitterscraper.logger.debug "Fetch #{@proxies.size} proxies"
+      else
+        @proxies = []
+        Twitterscraper.logger.debug 'Proxy disabled'
+      end
+    end
+
+    def request_headers
+      @request_headers
     end
 
     def cache_enabled?
@@ -14,5 +36,9 @@ module Twitterscraper
     def proxy_enabled?
       @proxy
     end
+
+    def proxies
+      @proxies
+    end
   end
 end

data/lib/twitterscraper/query.rb

@@ -10,14 +10,6 @@ module Twitterscraper
   module Query
     include Logger
 
-    USER_AGENT_LIST = [
-        'Mozilla/5.0 (Windows; U; Windows NT 6.1; x64; fr; rv:1.9.2.13) Gecko/20101203 Firebird/3.6.13',
-        'Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko',
-        'Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:2.2) Gecko/20110201',
-        'Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16',
-        'Mozilla/5.0 (Windows NT 5.2; RW; rv:7.0a1) Gecko/20091211 SeaMonkey/9.23a1pre',
-    ]
-
     INIT_URL = 'https://twitter.com/search?f=tweets&vertical=default&q=__QUERY__&l=__LANG__'
     RELOAD_URL = 'https://twitter.com/i/search/timeline?f=tweets&vertical=' +
         'default&include_available_features=1&include_entities=1&' +
@@ -43,13 +35,13 @@ module Twitterscraper
       end
     end
 
-    def get_single_page(url, headers, proxies, timeout = 6, retries = 30)
+    def get_single_page(url, timeout = 6, retries = 30)
       return nil if stop_requested?
-      unless proxies.empty?
+      if proxy_enabled?
         proxy = proxies.sample
         logger.info("Using proxy #{proxy}")
       end
-      Http.get(url, headers, proxy, timeout)
+      Http.get(url, request_headers, proxy, timeout)
     rescue => e
       logger.debug "get_single_page: #{e.inspect}"
       if (retries -= 1) > 0
@@ -74,24 +66,28 @@ module Twitterscraper
       [items_html, json_resp]
     end
 
-    def query_single_page(query, lang, type, pos, headers: [], proxies: [])
+    def query_single_page(query, lang, type, pos)
       logger.info "Querying #{query}"
-      query = ERB::Util.url_encode(query)
+      encoded_query = ERB::Util.url_encode(query)
 
-      url = build_query_url(query, lang, type, pos)
+      url = build_query_url(encoded_query, lang, type, pos)
       http_request = lambda do
-        logger.debug "Scraping tweets from #{url}"
-        get_single_page(url, headers, proxies)
+        logger.debug "Scraping tweets from url=#{url}"
+        get_single_page(url)
       end
 
       if cache_enabled?
         client = Cache.new
         if (response = client.read(url))
-          logger.debug 'Fetching tweets from cache'
+          logger.debug "Fetching tweets from cache url=#{url}"
         else
           response = http_request.call
           client.write(url, response) unless stop_requested?
         end
+        if @queries && query == @queries.last && pos.nil?
+          logger.debug "Delete a cache query=#{query}"
+          client.delete(url)
+        end
       else
         response = http_request.call
       end
@@ -135,132 +131,151 @@ module Twitterscraper
       if start_date && end_date
         if start_date == end_date
           raise Error.new('Please specify different values for :start_date and :end_date.')
-        elsif start_date > end_date
+        elsif Date.parse(start_date) > Date.parse(end_date)
           raise Error.new(':start_date must occur before :end_date.')
         end
       end
 
       if start_date
-        if start_date < OLDEST_DATE
+        if Date.parse(start_date) < OLDEST_DATE
           raise Error.new(":start_date must be greater than or equal to #{OLDEST_DATE}")
         end
       end
     end
 
-    def build_queries(query, start_date, end_date)
-      if start_date && end_date
-        # date_range = start_date.upto(end_date - 1)
-        # date_range.map { |date| query + " since:#{date} until:#{date + 1}" }
+    def build_queries(query, start_date, end_date, threads_granularity, type)
+      if type.search?
+        start_date = Date.parse(start_date) if start_date.is_a?(String)
+        end_date = Date.parse(end_date) if end_date.is_a?(String)
+      elsif type.user?
+        start_date = nil
+        end_date = nil
+      end
 
-        queries = []
-        time = Time.utc(start_date.year, start_date.month, start_date.day, 0, 0, 0)
-        end_time = Time.utc(end_date.year, end_date.month, end_date.day, 0, 0, 0)
+      if start_date && end_date
+        if threads_granularity == 'auto'
+          threads_granularity = start_date.upto(end_date - 1).to_a.size >= 28 ? 'day' : 'hour'
+        end
 
-        while true
-          if time < Time.now.utc
-            queries << (query + " since:#{time.strftime('%Y-%m-%d_%H:00:00')}_UTC until:#{(time + 3600).strftime('%Y-%m-%d_%H:00:00')}_UTC")
+        if threads_granularity == 'day'
+          date_range = start_date.upto(end_date - 1)
+          queries = date_range.map { |date| query + " since:#{date}_00:00:00_UTC until:#{date + 1}_00:00:00_UTC" }
+        elsif threads_granularity == 'hour'
+          time = Time.utc(start_date.year, start_date.month, start_date.day, 0, 0, 0)
+          end_time = Time.utc(end_date.year, end_date.month, end_date.day, 0, 0, 0)
+          queries = []
+
+          while true
+            if time < Time.now.utc
+              queries << (query + " since:#{time.strftime('%Y-%m-%d_%H')}:00:00_UTC until:#{(time + 3600).strftime('%Y-%m-%d_%H')}:00:00_UTC")
+            end
+            time += 3600
+            break if time >= end_time
           end
-          time += 3600
-          break if time >= end_time
+        else
+          raise Error.new("Invalid :threads_granularity value=#{threads_granularity}")
         end
 
-        queries
+        @queries = queries
 
       elsif start_date
-        [query + " since:#{start_date}"]
+        [query + " since:#{start_date}_00:00:00_UTC"]
       elsif end_date
-        [query + " until:#{end_date}"]
+        [query + " until:#{end_date}_00:00:00_UTC"]
       else
         [query]
       end
     end
 
-    def main_loop(query, lang, type, limit, daily_limit, headers, proxies)
+    def main_loop(query, lang, type, limit, daily_limit)
       pos = nil
-      daily_tweets = []
+      tmp_tweets = []
 
       while true
-        new_tweets, new_pos = query_single_page(query, lang, type, pos, headers: headers, proxies: proxies)
+        new_tweets, new_pos = query_single_page(query, lang, type, pos)
         unless new_tweets.empty?
-          daily_tweets.concat(new_tweets)
-          daily_tweets.uniq! { |t| t.tweet_id }
+          tmp_tweets.concat(new_tweets)
+          tmp_tweets.uniq! { |t| t.tweet_id }
+        end
 
-          @mutex.synchronize {
-            @all_tweets.concat(new_tweets)
-            @all_tweets.uniq! { |t| t.tweet_id }
-          }
+        @results_counter[Parallel.worker_number] = tmp_tweets.size
+        total_size = @all_tweets.size + @results_counter.values.sum
+        logger.info "Got tweets new=#{new_tweets.size} tmp=#{tmp_tweets.size} all=#{@all_tweets.size} total=#{total_size}"
+
+        if !@stop_requested && total_size >= limit
+          logger.warn "The limit you specified has been reached limit=#{limit} tweets=#{total_size}"
+          @stop_requested = true
         end
-        logger.info "Got #{new_tweets.size} tweets (total #{@all_tweets.size})"
 
         break unless new_pos
-        break if daily_limit && daily_tweets.size >= daily_limit
+        break if @stop_requested
+        break if daily_limit && tmp_tweets.size >= daily_limit
         break if @all_tweets.size >= limit
 
         pos = new_pos
       end
 
-      if !@stop_requested && @all_tweets.size >= limit
-        logger.warn "The limit you specified has been reached limit=#{limit} tweets=#{@all_tweets.size}"
-        @stop_requested = true
-      end
+      tmp_tweets
     end
 
     def stop_requested?
       @stop_requested
     end
 
-    def query_tweets(query, type: 'search', start_date: nil, end_date: nil, lang: nil, limit: 100, daily_limit: nil, order: 'desc', threads: 2)
-      start_date = Date.parse(start_date) if start_date && start_date.is_a?(String)
-      end_date = Date.parse(end_date) if end_date && end_date.is_a?(String)
-      queries = build_queries(query, start_date, end_date)
+    def query_tweets(query, type: 'search', start_date: nil, end_date: nil, lang: nil, limit: 100, daily_limit: nil, order: 'desc', threads: 10, threads_granularity: 'auto')
       type = Type.new(type)
+      queries = build_queries(query, start_date, end_date, threads_granularity, type)
       if threads > queries.size
         threads = queries.size
       end
-      if proxy_enabled?
-        proxies = Proxy::Pool.new
-        logger.debug "Fetch #{proxies.size} proxies"
-      else
-        proxies = []
-        logger.debug 'Proxy disabled'
-      end
       logger.debug "Cache #{cache_enabled? ? 'enabled' : 'disabled'}"
 
       validate_options!(queries, type: type, start_date: start_date, end_date: end_date, lang: lang, limit: limit, threads: threads)
 
+      logger.info "The number of queries #{queries.size}"
       logger.info "The number of threads #{threads}"
 
-      headers = {'User-Agent': USER_AGENT_LIST.sample, 'X-Requested-With': 'XMLHttpRequest'}
-      logger.info "Headers #{headers}"
-
       @all_tweets = []
-      @mutex = Mutex.new
       @stop_requested = false
+      @results_counter = {}
 
       if threads > 1
+        @mutex = Mutex.new
         Thread.abort_on_exception = true
         logger.debug "Set 'Thread.abort_on_exception' to true"
 
         Parallel.each(queries, in_threads: threads) do |query|
-          main_loop(query, lang, type, limit, daily_limit, headers, proxies)
+          @results_counter[Parallel.worker_number] = 0
+          tmp_tweets = main_loop(query, lang, type, limit, daily_limit)
+          @mutex.synchronize {
+            @all_tweets.concat(tmp_tweets)
+            @all_tweets.uniq! { |t| t.tweet_id }
+          }
+          @results_counter[Parallel.worker_number] = 0
+
           raise Parallel::Break if stop_requested?
         end
       else
         queries.each do |query|
-          main_loop(query, lang, type, limit, daily_limit, headers, proxies)
+          tmp_tweets = main_loop(query, lang, type, limit, daily_limit)
+          @all_tweets.concat(tmp_tweets)
+          @all_tweets.uniq! { |t| t.tweet_id }
+
           break if stop_requested?
         end
       end
 
+      logger.info "Return #{@all_tweets.size} tweets"
+
       @all_tweets.sort_by { |tweet| (order == 'desc' ? -1 : 1) * tweet.created_at.to_i }
     end
 
-    def search(query, start_date: nil, end_date: nil, lang: '', limit: 100, daily_limit: nil, order: 'desc', threads: 2)
-      query_tweets(query, type: 'search', start_date: start_date, end_date: end_date, lang: lang, limit: limit, daily_limit: daily_limit, order: order, threads: threads)
+    def search(query, start_date: nil, end_date: nil, lang: '', limit: 100, daily_limit: nil, order: 'desc', threads: 10, threads_granularity: 'auto')
+      query_tweets(query, type: 'search', start_date: start_date, end_date: end_date, lang: lang, limit: limit, daily_limit: daily_limit, order: order, threads: threads, threads_granularity: threads_granularity)
     end
 
     def user_timeline(screen_name, limit: 100, order: 'desc')
-      query_tweets(screen_name, type: 'user', start_date: nil, end_date: nil, lang: nil, limit: limit, daily_limit: nil, order: order, threads: 1)
+      query_tweets(screen_name, type: 'user', start_date: nil, end_date: nil, lang: nil, limit: limit, daily_limit: nil, order: order, threads: 1, threads_granularity: nil)
     end
   end
 end

data/lib/twitterscraper/template.rb

@@ -4,25 +4,53 @@ module Twitterscraper
       path = File.join(File.dirname(__FILE__), 'template/tweets.html.erb')
       template = ERB.new(File.read(path))
 
+      tweets = tweets.sort_by { |t| t.created_at.to_i }
+      grouping = options['chart_grouping'] || 'auto'
+
       template.result_with_hash(
           chart_name: name,
-          chart_data: chart_data(tweets).to_json,
-          first_tweet: tweets.sort_by { |t| t.created_at.to_i }[0],
-          last_tweet: tweets.sort_by { |t| t.created_at.to_i }[-1],
-          tweets_size: tweets.size,
-          tweets: tweets.take(50)
+          chart_data: chart_data(tweets, grouping: grouping).to_json,
+          first_tweet: tweets[0],
+          last_tweet: tweets[-1],
+          tweets: tweets,
+          convert_limit: 30,
       )
     end
 
-    def chart_data(tweets)
+    def chart_data(tweets, grouping: 'auto')
+      if grouping && tweets.size > 100
+        if grouping == 'auto'
+          month = 28 * 24 * 60 * 60 # 28 days
+          duration = tweets[-1].created_at - tweets[0].created_at
+
+          if duration > 3 * month
+            grouping = 'day'
+          elsif duration > month || tweets.size > 10000
+            grouping = 'hour'
+          else
+            grouping = 'minute'
+          end
+        end
+      end
+
+      Twitterscraper.logger.info "Chart grouping #{grouping}"
+
       data = tweets.each_with_object(Hash.new(0)) do |tweet, memo|
         t = tweet.created_at
-        min = (t.min.to_f / 5).floor * 5
-        time = Time.new(t.year, t.month, t.day, t.hour, min, 0, '+00:00')
+
+        if grouping == 'day'
+          time = Time.new(t.year, t.month, t.day, 0, 0, 0, '+00:00')
+        elsif grouping == 'hour'
+          time = Time.new(t.year, t.month, t.day, t.hour, 0, 0, '+00:00')
+        elsif grouping == 'minute'
+          time = Time.new(t.year, t.month, t.day, t.hour, t.min, 0, '+00:00')
+        else
+          time = t
+        end
         memo[time.to_i] += 1
       end
 
-      data.sort_by { |k, v| k }.map do |timestamp, count|
+      data.sort_by { |k, _| k }.map do |timestamp, count|
         [timestamp * 1000, count]
       end
     end

data/lib/twitterscraper/template/tweets.html.erb

@@ -1,27 +1,30 @@
-<html>
+<!DOCTYPE html>
+<html lang="ja">
 <head>
-  <script>
-      window.twttr = (function (d, s, id) {
-          var js, fjs = d.getElementsByTagName(s)[0], t = window.twttr || {};
-          if (d.getElementById(id)) return t;
-          js = d.createElement(s);
-          js.id = id;
-          js.src = "https://platform.twitter.com/widgets.js";
-          fjs.parentNode.insertBefore(js, fjs);
-
-          t._e = [];
-          t.ready = function (f) {
-              t._e.push(f);
-          };
-
-          return t;
-      }(document, "script", "twitter-wjs"));
-  </script>
+  <meta charset="UTF-8">
 
   <script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.27.0/moment.min.js" integrity="sha512-rmZcZsyhe0/MAjquhTgiUcb4d9knaFc7b5xAfju483gbEXTkeJRUMIPk6s3ySZMYUHEcjKbjLjyddGWMrNEvZg==" crossorigin="anonymous"></script>
   <script src="https://cdnjs.cloudflare.com/ajax/libs/moment-timezone/0.5.31/moment-timezone-with-data.min.js" integrity="sha512-HZcf3uHWA+Y2P5KNv+F/xa87/flKVP92kUTe/KXjU8URPshczF1Dx+cL5bw0VBGhmqWAK0UbhcqxBbyiNtAnWQ==" crossorigin="anonymous"></script>
   <script src="https://code.highcharts.com/stock/highstock.js"></script>
   <script>
+      function updateTweets() {
+          window.twttr = (function (d, s, id) {
+              var js, fjs = d.getElementsByTagName(s)[0], t = window.twttr || {};
+              if (d.getElementById(id)) return t;
+              js = d.createElement(s);
+              js.id = id;
+              js.src = "https://platform.twitter.com/widgets.js";
+              fjs.parentNode.insertBefore(js, fjs);
+
+              t._e = [];
+              t.ready = function (f) {
+                  t._e.push(f);
+              };
+
+              return t;
+          }(document, "script", "twitter-wjs"));
+      }
+
       function drawChart() {
           Highcharts.setOptions({
               time: {
@@ -29,30 +32,43 @@
               }
           });
 
-          Highcharts.stockChart('chart', {
+          var data = <%= chart_data %>;
+          var config = {
               title: {
-                  text: '<%= tweets_size %> tweets of <%= chart_name %>'
+                  text: '<%= tweets.size %> tweets of <%= chart_name %>'
               },
               subtitle: {
-                  text: 'since:<%= first_tweet.created_at.localtime %> until:<%= last_tweet.created_at.localtime %>'
+                  text: 'since:<%= first_tweet.created_at.localtime.strftime('%Y-%m-%d %H:%M') %> until:<%= last_tweet.created_at.localtime.strftime('%Y-%m-%d %H:%M') %>'
               },
               series: [{
-                  data: <%= chart_data %>
+                  data: data
               }],
               rangeSelector: {enabled: false},
               scrollbar: {enabled: false},
               navigator: {enabled: false},
               exporting: {enabled: false},
               credits: {enabled: false}
-          });
+          };
+
+          Highcharts.stockChart('chart-container', config);
       }
 
       document.addEventListener("DOMContentLoaded", function () {
           drawChart();
+          updateTweets();
       });
   </script>
 
   <style type=text/css>
+    #chart-container {
+      max-width: 1200px;
+      height: 675px;
+      margin: 0 auto;
+      border: 1px solid rgb(204, 214, 221);
+      display: flex;
+      justify-content: center;
+      align-items: center;
+    }
     .tweets-container {
       max-width: 550px;
       margin: 0 auto 0 auto;
@@ -64,17 +80,31 @@
   </style>
 </head>
 <body>
-<div id="chart"></div>
+<div id="chart-container"><div style="color: gray;">Loading...</div></div>
 
 <div class="tweets-container">
-  <% tweets.each do |tweet| %>
-    <blockquote class="twitter-tweet">
-      <a href="<%= tweet.tweet_url %>"></a>
-    </blockquote>
-  <% end %>
+  <% tweets.sort_by { |t| -t.created_at.to_i }.take(1000).each.with_index do |tweet, i| %>
+    <% tweet_time = tweet.created_at.localtime.strftime('%Y-%m-%d %H:%M') %>
+    <% if i < convert_limit %>
+      <blockquote class="twitter-tweet">
+    <% else %>
+      <div class="twitter-tweet" style="border: 1px solid rgb(204, 214, 221);">
+    <% end %>
+
+    <div style="display: grid; grid-template-rows: 24px 24px; grid-template-columns: 48px 1fr;">
+      <div style="grid-row: 1/3; grid-column:  1/2;"><img src="<%= tweet.profile_image_url %>" width="48" height="48" loading="lazy"></div>
+      <div style="grid-row: 1/2; grid-column:  2/3;"><%= tweet.name %></div>
+      <div style="grid-row: 2/3; grid-column:  2/3;"><a href="https://twitter.com/<%= tweet.screen_name %>">@<%= tweet.screen_name %></a></div>
+    </div>
+
+    <div><%= tweet.text %></div>
+    <div><a href="<%= tweet.tweet_url %>"><small><%= tweet_time %></small></a></div>
 
-  <% if tweets_size > tweets.size %>
-    <div>and more!</div>
+    <% if i < convert_limit %>
+      </blockquote>
+    <% else %>
+      </div>
+    <% end %>
   <% end %>
 </div>
 

data/lib/twitterscraper/tweet.rb

@@ -6,6 +6,7 @@ module Twitterscraper
         :screen_name,
         :name,
         :user_id,
+        :profile_image_url,
         :tweet_id,
         :text,
         :links,
@@ -51,6 +52,11 @@ module Twitterscraper
         end
       end
 
+      # .js-stream-item
+      #   .js-stream-tweet{data: {screen-name:, tweet-id:}}
+      #     .stream-item-header
+      #     .js-tweet-text-container
+      #     .stream-item-footer
       def from_html(text)
         html = Nokogiri::HTML(text)
         from_tweets_html(html.xpath("//li[@class[contains(., 'js-stream-item')]]/div[@class[contains(., 'js-stream-tweet')]]"))
@@ -72,6 +78,8 @@ module Twitterscraper
         end
 
         inner_html = Nokogiri::HTML(html.inner_html)
+
+        profile_image_url = inner_html.xpath("//img[@class[contains(., 'js-action-profile-avatar')]]").first.attr('src').gsub(/_bigger/, '')
         text = inner_html.xpath("//div[@class[contains(., 'js-tweet-text-container')]]/p[@class[contains(., 'js-tweet-text')]]").first.text
         links = inner_html.xpath("//a[@class[contains(., 'twitter-timeline-link')]]").map { |elem| elem.attr('data-expanded-url') }.select { |link| link && !link.include?('pic.twitter') }
         image_urls = inner_html.xpath("//div[@class[contains(., 'AdaptiveMedia-photoContainer')]]").map { |elem| elem.attr('data-image-url') }
@@ -99,6 +107,7 @@ module Twitterscraper
             screen_name: screen_name,
             name: html.attr('data-name'),
             user_id: html.attr('data-user-id').to_i,
+            profile_image_url: profile_image_url,
             tweet_id: tweet_id,
             text: text,
             links: links,

data/lib/twitterscraper/type.rb

@@ -11,5 +11,9 @@ module Twitterscraper
     def user?
       @value == 'user'
     end
+
+    def to_s
+      @value
+    end
   end
 end

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Twitterscraper
-  VERSION = '0.16.0'
+  VERSION = '0.20.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twitterscraper-ruby
 version: !ruby/object:Gem::Version
-  version: 0.16.0
+  version: 0.20.1
 platform: ruby
 authors:
 - ts-3156
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-18 00:00:00.000000000 Z
+date: 2021-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri