twitterscraper-ruby 0.14.0 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf902c947e866cc99e79fbb9f8a51c829accd44aed03ef7657562bf41932c73d
-  data.tar.gz: 1bc5a0698a17b244ee9228d7728767dd00218179a5a49e0852a74cc722322ef0
+  metadata.gz: ac0c10b18d836983cc6b73e25b9ed333af2f620106a07c6bc6a40058fb127895
+  data.tar.gz: e6fc18219d9127fb30ba57e39dc4656c0f0a3c108428d959de5bac9e7d317088
 SHA512:
-  metadata.gz: 629de8698af1391c210b496e9aadb51ad5f9d7157b1be5d0aa669ae821671e2b5624ba51083fb14b61f93618ff3e90aea1ac0eccb6ea00360fac48a2dfc436c7
-  data.tar.gz: 3f3706bee5f2a92a2addae034201e2e8cee3fef43efdc323be963cbaf1b94c31c53aa49a19e58a068498722dfe07e9796e097fb04364a9afda56d06132e6b935
+  metadata.gz: 90cbf06b606878dc36b4bba44669139c273bf03b08a777ad87036834841bcb4b052e0559813dc56e4be124442abfc5a7fc44c5c9524c74929ca02b1d287d346b
+  data.tar.gz: ada0b74ee42ff62964b73ad9b49358227cdaf4fc87420cf12cf65af95168ad9775615a504345ebc83d3b791e9c0d892691c55bc477eddd647b3e8934f752fb9c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    twitterscraper-ruby (0.14.0)
+    twitterscraper-ruby (0.17.0)
       nokogiri
       parallel
 

data/README.md

@@ -5,15 +5,17 @@
 
 A gem to scrape https://twitter.com/search. This gem is inspired by [taspinar/twitterscraper](https://github.com/taspinar/twitterscraper).
 
+Please feel free to ask [@ts_3156](https://twitter.com/ts_3156) if you have any questions.
+
 
 ## Twitter Search API vs. twitterscraper-ruby
 
-### Twitter Search API
+#### Twitter Search API
 
 - The number of tweets: 180 - 450 requests/15 minutes (18,000 - 45,000 tweets/15 minutes)
 - The time window: the past 7 days
 
-### twitterscraper-ruby
+#### twitterscraper-ruby
 
 - The number of tweets: Unlimited
 - The time window: from 2006-3-21 to today
@@ -30,48 +32,98 @@ $ gem install twitterscraper-ruby
 
 ## Usage
 
-Command-line interface:
+#### Command-line interface:
+
+Returns a collection of relevant tweets matching a specified query.
 
 ```shell script
-$ twitterscraper --query KEYWORD --start_date 2020-06-01 --end_date 2020-06-30 --lang ja \
-      --limit 100 --threads 10 --output output.json
+$ twitterscraper --type search --query KEYWORD --start_date 2020-06-01 --end_date 2020-06-30 --lang ja \
+      --limit 100 --threads 10 --output tweets.json
 ```
 
-From Within Ruby:
+Returns a collection of the most recent tweets posted by the user indicated by the screen_name
+
+```shell script
+$ twitterscraper --type user --query SCREEN_NAME --limit 100 --output tweets.json
+```
+
+#### From Within Ruby:
 
 ```ruby
 require 'twitterscraper'
+client = Twitterscraper::Client.new(cache: true, proxy: true)
+```
 
-options = {
-  start_date: '2020-06-01',
-  end_date:   '2020-06-30',
-  lang:       'ja',
-  limit:      100,
-  threads:    10,
-}
+Returns a collection of relevant tweets matching a specified query.
 
-client = Twitterscraper::Client.new(cache: true, proxy: true)
-tweets = client.query_tweets(KEYWORD, options)
+```ruby
+tweets = client.search(KEYWORD, start_date: '2020-06-01', end_date: '2020-06-30', lang: 'ja', limit: 100, threads: 10)
+```
+
+Returns a collection of the most recent tweets posted by the user indicated by the screen_name
+
+```ruby
+tweets = client.user_timeline(SCREEN_NAME, limit: 100)
+```
+
+
+## Examples
+
+```shell script
+$ twitterscraper --query twitter --limit 1000
+$ cat tweets.json | jq . | less
+```
+
+
+## Attributes
 
+### Tweet
+
+```ruby
 tweets.each do |tweet|
   puts tweet.tweet_id
   puts tweet.text
   puts tweet.tweet_url
   puts tweet.created_at
 
+  attr_names = hash.keys
   hash = tweet.attrs
-  puts hash.keys
+  json = tweet.to_json
 end
 ```
 
-
-## Attributes
-
-### Tweet
+```json
+[
+  {
+      "screen_name": "@name",
+      "name": "Name",
+      "user_id": 12340000,
+      "profile_image_url": "https://pbs.twimg.com/profile_images/1826000000/0000.png",
+      "tweet_id": 1234000000000000,
+      "text": "Thanks Twitter!",
+      "links": [],
+      "hashtags": [],
+      "image_urls": [],
+      "video_url": null,
+      "has_media": null,
+      "likes": 10,
+      "retweets": 20,
+      "replies": 0,
+      "is_replied": false,
+      "is_reply_to": false,
+      "parent_tweet_id": null,
+      "reply_to_users": [],
+      "tweet_url": "https://twitter.com/name/status/1234000000000000",
+      "timestamp": 1594793000,
+      "created_at": "2020-07-15 00:00:00 +0000"
+    }
+]
+```
 
 - screen_name
 - name
 - user_id
+- profile_image_url
 - tweet_id
 - text
 - links
@@ -110,44 +162,25 @@ end
 Search operators documentation is in [Standard search operators](https://developer.twitter.com/en/docs/tweets/rules-and-filtering/overview/standard-operators).
 
 
-## Examples
-
-```shell script
-$ twitterscraper --query twitter --limit 1000
-$ cat tweets.json | jq . | less
-```
-
-```json
-[
-  {
-    "screen_name": "@screenname",
-    "name": "name",
-    "user_id": 1194529546483000000,
-    "tweet_id": 1282659891992000000,
-    "tweet_url": "https://twitter.com/screenname/status/1282659891992000000",
-    "created_at": "2020-07-13 12:00:00 +0000",
-    "text": "Thanks Twitter!"
-  }
-]
-```
-
 ## CLI Options
 
-| Option | Description | Default |
-| ------------- | ------------- | ------------- |
-| `-h`, `--help` | This option displays a summary of twitterscraper. | |
-| `--query` | Specify a keyword used during the search. | |
-| `--start_date` | Used as "since:yyyy-mm-dd for your query. This means "since the date". | |
-| `--end_date` | Used as "until:yyyy-mm-dd for your query. This means "before the date". | |
-| `--lang` | Retrieve tweets written in a specific language. | |
-| `--limit` | Stop scraping when *at least* the number of tweets indicated with --limit is scraped. | 100 |
-| `--order` | Sort order of the results. | desc |
-| `--threads` | Set the number of threads twitterscraper-ruby should initiate while scraping for your query. | 2 |
-| `--proxy` | Scrape https://twitter.com/search via proxies. | true |
-| `--cache` | Enable caching. | true |
-| `--format` | The format of the output. | json |
-| `--output` | The name of the output file. | tweets.json |
-| `--verbose` | Print debug messages. | tweets.json |
+| Option | Type | Description | Value |
+| ------------- | ------------- | ------------- | ------------- |
+| `--help`       | string  | This option displays a summary of twitterscraper. | |
+| `--type`       | string  | Specify a search type. | search(default) or user |
+| `--query`      | string  | Specify a keyword used during the search. | |
+| `--start_date` | string  | Used as "since:yyyy-mm-dd for your query. This means "since the date". | |
+| `--end_date`   | string  | Used as "until:yyyy-mm-dd for your query. This means "before the date". | |
+| `--lang`       | string  | Retrieve tweets written in a specific language. | |
+| `--limit`      | integer | Stop scraping when *at least* the number of tweets indicated with --limit is scraped. | 100 |
+| `--order`      | string  | Sort a order of the results. | desc(default) or asc |
+| `--threads`    | integer | Set the number of threads twitterscraper-ruby should initiate while scraping for your query. | 2 |
+| `--threads_granularity` | string | | auto |
+| `--proxy`      | boolean | Scrape https://twitter.com/search via proxies. | true(default) or false |
+| `--cache`      | boolean | Enable caching. | true(default) or false |
+| `--format`     | string  | The format of the output. | json(default) or html |
+| `--output`     | string  | The name of the output file. | tweets.json |
+| `--verbose`    |         | Print debug messages. | |
 
 
 ## Contributing

data/lib/twitterscraper.rb

@@ -4,6 +4,7 @@ require 'twitterscraper/http'
 require 'twitterscraper/lang'
 require 'twitterscraper/cache'
 require 'twitterscraper/query'
+require 'twitterscraper/type'
 require 'twitterscraper/client'
 require 'twitterscraper/tweet'
 require 'twitterscraper/template'

data/lib/twitterscraper/cache.rb

@@ -4,7 +4,7 @@ require 'digest/md5'
 module Twitterscraper
   class Cache
     def initialize()
-      @ttl = 3600 # 1 hour
+      @ttl = 86400 # 1 day
       @dir = 'cache'
       Dir.mkdir(@dir) unless File.exist?(@dir)
     end
@@ -25,6 +25,12 @@ module Twitterscraper
       File.write(file, entry.to_json)
     end
 
+    def delete(key)
+      key = cache_key(key)
+      file = File.join(@dir, key)
+      File.delete(file) if File.exist?(file)
+    end
+
     def fetch(key, &block)
       if (value = read(key))
         value

data/lib/twitterscraper/cli.rb

@@ -16,6 +16,7 @@ module Twitterscraper
       print_version || return if print_version?
 
       query_options = {
+          type: options['type'],
           start_date: options['start_date'],
           end_date: options['end_date'],
           lang: options['lang'],
@@ -23,19 +24,20 @@ module Twitterscraper
           daily_limit: options['daily_limit'],
           order: options['order'],
           threads: options['threads'],
+          threads_granularity: options['threads_granularity'],
       }
       client = Twitterscraper::Client.new(cache: options['cache'], proxy: options['proxy'])
       tweets = client.query_tweets(options['query'], query_options)
-      export(tweets) unless tweets.empty?
+      export(options['query'], tweets) unless tweets.empty?
     end
 
-    def export(tweets)
+    def export(name, tweets)
       write_json = lambda { File.write(options['output'], generate_json(tweets)) }
 
       if options['format'] == 'json'
         write_json.call
       elsif options['format'] == 'html'
-        File.write('tweets.html', Template.tweets_embedded_html(tweets))
+        File.write(options['output'], Template.new.tweets_embedded_html(name, tweets, options))
       else
         write_json.call
       end
@@ -59,6 +61,7 @@ module Twitterscraper
           'help',
           'v',
           'version',
+          'type:',
           'query:',
           'start_date:',
           'end_date:',
@@ -67,6 +70,7 @@ module Twitterscraper
           'daily_limit:',
           'order:',
           'threads:',
+          'threads_granularity:',
           'output:',
           'format:',
           'cache:',
@@ -75,14 +79,16 @@ module Twitterscraper
           'verbose',
       )
 
+      options['type'] ||= 'search'
       options['start_date'] = Query::OLDEST_DATE if options['start_date'] == 'oldest'
       options['lang'] ||= ''
       options['limit'] = (options['limit'] || 100).to_i
       options['daily_limit'] = options['daily_limit'].to_i if options['daily_limit']
-      options['threads'] = (options['threads'] || 2).to_i
+      options['threads'] = (options['threads'] || 10).to_i
+      options['threads_granularity'] ||= 'auto'
       options['format'] ||= 'json'
       options['order'] ||= 'desc'
-      options['output'] ||= "tweets.#{options['format']}"
+      options['output'] ||= build_output_name(options)
 
       options['cache'] = options['cache'] != 'false'
       options['proxy'] = options['proxy'] != 'false'
@@ -90,6 +96,12 @@ module Twitterscraper
       options
     end
 
+    def build_output_name(options)
+      query = ERB::Util.url_encode(options['query'])
+      date = [options['start_date'], options['end_date']].select { |val| val && !val.empty? }.join('_')
+      [options['type'], 'tweets', date, query].compact.join('_') + '.' + options['format']
+    end
+
     def initialize_logger
       Twitterscraper.logger.level = ::Logger::DEBUG if options['verbose']
     end

data/lib/twitterscraper/query.rb

@@ -22,23 +22,24 @@ module Twitterscraper
     RELOAD_URL = 'https://twitter.com/i/search/timeline?f=tweets&vertical=' +
         'default&include_available_features=1&include_entities=1&' +
         'reset_error_state=false&src=typd&max_position=__POS__&q=__QUERY__&l=__LANG__'
-    INIT_URL_USER = 'https://twitter.com/{u}'
-    RELOAD_URL_USER = 'https://twitter.com/i/profiles/show/{u}/timeline/tweets?' +
+    INIT_URL_USER = 'https://twitter.com/__USER__'
+    RELOAD_URL_USER = 'https://twitter.com/i/profiles/show/__USER__/timeline/tweets?' +
         'include_available_features=1&include_entities=1&' +
-        'max_position={pos}&reset_error_state=false'
-
-    def build_query_url(query, lang, pos, from_user = false)
-      # if from_user
-      #   if !pos
-      #     INIT_URL_USER.format(u = query)
-      #   else
-      #     RELOAD_URL_USER.format(u = query, pos = pos)
-      #   end
-      # end
-      if pos
-        RELOAD_URL.sub('__QUERY__', query).sub('__LANG__', lang.to_s).sub('__POS__', pos)
+        'max_position=__POS__&reset_error_state=false'
+
+    def build_query_url(query, lang, type, pos)
+      if type.user?
+        if pos
+          RELOAD_URL_USER.sub('__USER__', query).sub('__POS__', pos.to_s)
+        else
+          INIT_URL_USER.sub('__USER__', query)
+        end
       else
-        INIT_URL.sub('__QUERY__', query).sub('__LANG__', lang.to_s)
+        if pos
+          RELOAD_URL.sub('__QUERY__', query).sub('__LANG__', lang.to_s).sub('__POS__', pos)
+        else
+          INIT_URL.sub('__QUERY__', query).sub('__LANG__', lang.to_s)
+        end
       end
     end
 
@@ -50,7 +51,7 @@ module Twitterscraper
       end
       Http.get(url, headers, proxy, timeout)
     rescue => e
-      logger.debug "query_single_page: #{e.inspect}"
+      logger.debug "get_single_page: #{e.inspect}"
       if (retries -= 1) > 0
         logger.info "Retrying... (Attempts left: #{retries - 1})"
         retry
@@ -68,26 +69,25 @@ module Twitterscraper
       else
         json_resp = JSON.parse(text)
         items_html = json_resp['items_html'] || ''
-        logger.warn json_resp['message'] if json_resp['message'] # Sorry, you are rate limited.
       end
 
       [items_html, json_resp]
     end
 
-    def query_single_page(query, lang, pos, from_user = false, headers: [], proxies: [])
+    def query_single_page(query, lang, type, pos, headers: [], proxies: [])
       logger.info "Querying #{query}"
       query = ERB::Util.url_encode(query)
 
-      url = build_query_url(query, lang, pos, from_user)
+      url = build_query_url(query, lang, type, pos)
       http_request = lambda do
-        logger.debug "Scraping tweets from #{url}"
+        logger.debug "Scraping tweets from url=#{url}"
         get_single_page(url, headers, proxies)
       end
 
       if cache_enabled?
         client = Cache.new
         if (response = client.read(url))
-          logger.debug 'Fetching tweets from cache'
+          logger.debug "Fetching tweets from cache url=#{url}"
         else
           response = http_request.call
           client.write(url, response) unless stop_requested?
@@ -99,6 +99,12 @@ module Twitterscraper
 
       html, json_resp = parse_single_page(response, pos.nil?)
 
+      if json_resp && json_resp['message']
+        logger.warn json_resp['message'] # Sorry, you are rate limited.
+        @stop_requested = true
+        Cache.new.delete(url) if cache_enabled?
+      end
+
       tweets = Tweet.from_html(html)
 
       if tweets.empty?
@@ -107,8 +113,8 @@ module Twitterscraper
 
       if json_resp
         [tweets, json_resp['min_position']]
-      elsif from_user
-        raise NotImplementedError
+      elsif type.user?
+        [tweets, tweets[-1].tweet_id]
       else
         [tweets, "TWEET-#{tweets[-1].tweet_id}-#{tweets[0].tweet_id}"]
       end
@@ -116,7 +122,7 @@ module Twitterscraper
 
     OLDEST_DATE = Date.parse('2006-03-21')
 
-    def validate_options!(queries, start_date:, end_date:, lang:, limit:, threads:)
+    def validate_options!(queries, type:, start_date:, end_date:, lang:, limit:, threads:)
       query = queries[0]
       if query.nil? || query == ''
         raise Error.new('Please specify a search query.')
@@ -139,19 +145,33 @@ module Twitterscraper
           raise Error.new(":start_date must be greater than or equal to #{OLDEST_DATE}")
         end
       end
-
-      if end_date
-        today = Date.today
-        if end_date > Date.today
-          raise Error.new(":end_date must be less than or equal to today(#{today})")
-        end
-      end
     end
 
-    def build_queries(query, start_date, end_date)
+    def build_queries(query, start_date, end_date, threads_granularity)
       if start_date && end_date
-        date_range = start_date.upto(end_date - 1)
-        date_range.map { |date| query + " since:#{date} until:#{date + 1}" }
+        if threads_granularity == 'auto'
+          threads_granularity = start_date.upto(end_date - 1).to_a.size >= 28 ? 'day' : 'hour'
+        end
+
+        if threads_granularity == 'day'
+          date_range = start_date.upto(end_date - 1)
+          queries = date_range.map { |date| query + " since:#{date} until:#{date + 1}" }
+        else
+          time = Time.utc(start_date.year, start_date.month, start_date.day, 0, 0, 0)
+          end_time = Time.utc(end_date.year, end_date.month, end_date.day, 0, 0, 0)
+          queries = []
+
+          while true
+            if time < Time.now.utc
+              queries << (query + " since:#{time.strftime('%Y-%m-%d_%H:00:00')}_UTC until:#{(time + 3600).strftime('%Y-%m-%d_%H:00:00')}_UTC")
+            end
+            time += 3600
+            break if time >= end_time
+          end
+        end
+
+        queries
+
       elsif start_date
         [query + " since:#{start_date}"]
       elsif end_date
@@ -161,12 +181,12 @@ module Twitterscraper
       end
     end
 
-    def main_loop(query, lang, limit, daily_limit, headers, proxies)
+    def main_loop(query, lang, type, limit, daily_limit, headers, proxies)
       pos = nil
       daily_tweets = []
 
       while true
-        new_tweets, new_pos = query_single_page(query, lang, pos, headers: headers, proxies: proxies)
+        new_tweets, new_pos = query_single_page(query, lang, type, pos, headers: headers, proxies: proxies)
         unless new_tweets.empty?
           daily_tweets.concat(new_tweets)
           daily_tweets.uniq! { |t| t.tweet_id }
@@ -195,12 +215,18 @@ module Twitterscraper
       @stop_requested
     end
 
-    def query_tweets(query, start_date: nil, end_date: nil, lang: '', limit: 100, daily_limit: nil, order: 'desc', threads: 2)
-      start_date = Date.parse(start_date) if start_date && start_date.is_a?(String)
-      end_date = Date.parse(end_date) if end_date && end_date.is_a?(String)
-      queries = build_queries(query, start_date, end_date)
+    def query_tweets(query, type: 'search', start_date: nil, end_date: nil, lang: nil, limit: 100, daily_limit: nil, order: 'desc', threads: 10, threads_granularity: 'auto')
+      type = Type.new(type)
+      if type.search?
+        start_date = Date.parse(start_date) if start_date && start_date.is_a?(String)
+        end_date = Date.parse(end_date) if end_date && end_date.is_a?(String)
+      elsif type.user?
+        start_date = nil
+        end_date = nil
+      end
+
+      queries = build_queries(query, start_date, end_date, threads_granularity)
       if threads > queries.size
-        logger.warn 'The maximum number of :threads is the number of dates between :start_date and :end_date.'
         threads = queries.size
       end
       if proxy_enabled?
@@ -212,9 +238,9 @@ module Twitterscraper
       end
       logger.debug "Cache #{cache_enabled? ? 'enabled' : 'disabled'}"
 
+      validate_options!(queries, type: type, start_date: start_date, end_date: end_date, lang: lang, limit: limit, threads: threads)
 
-      validate_options!(queries, start_date: start_date, end_date: end_date, lang: lang, limit: limit, threads: threads)
-
+      logger.info "The number of queries #{queries.size}"
       logger.info "The number of threads #{threads}"
 
       headers = {'User-Agent': USER_AGENT_LIST.sample, 'X-Requested-With': 'XMLHttpRequest'}
@@ -229,17 +255,27 @@ module Twitterscraper
         logger.debug "Set 'Thread.abort_on_exception' to true"
 
         Parallel.each(queries, in_threads: threads) do |query|
-          main_loop(query, lang, limit, daily_limit, headers, proxies)
+          main_loop(query, lang, type, limit, daily_limit, headers, proxies)
           raise Parallel::Break if stop_requested?
         end
       else
         queries.each do |query|
-          main_loop(query, lang, limit, daily_limit, headers, proxies)
+          main_loop(query, lang, type, limit, daily_limit, headers, proxies)
           break if stop_requested?
         end
       end
 
+      logger.info "Return #{@all_tweets.size} tweets"
+
       @all_tweets.sort_by { |tweet| (order == 'desc' ? -1 : 1) * tweet.created_at.to_i }
     end
+
+    def search(query, start_date: nil, end_date: nil, lang: '', limit: 100, daily_limit: nil, order: 'desc', threads: 10, threads_granularity: 'auto')
+      query_tweets(query, type: 'search', start_date: start_date, end_date: end_date, lang: lang, limit: limit, daily_limit: daily_limit, order: order, threads: threads, threads_granularity: threads_granularity)
+    end
+
+    def user_timeline(screen_name, limit: 100, order: 'desc')
+      query_tweets(screen_name, type: 'user', start_date: nil, end_date: nil, lang: nil, limit: limit, daily_limit: nil, order: order, threads: 1, threads_granularity: nil)
+    end
   end
 end

data/lib/twitterscraper/template.rb

@@ -1,48 +1,30 @@
 module Twitterscraper
-  module Template
-    module_function
+  class Template
+    def tweets_embedded_html(name, tweets, options)
+      path = File.join(File.dirname(__FILE__), 'template/tweets.html.erb')
+      template = ERB.new(File.read(path))
 
-    def tweets_embedded_html(tweets)
-      tweets_html = tweets.map { |t| EMBED_TWEET_HTML.sub('__TWEET_URL__', t.tweet_url) }
-      EMBED_TWEETS_HTML.sub('__TWEETS__', tweets_html.join)
+      template.result_with_hash(
+          chart_name: name,
+          chart_data: chart_data(tweets).to_json,
+          first_tweet: tweets.sort_by { |t| t.created_at.to_i }[0],
+          last_tweet: tweets.sort_by { |t| t.created_at.to_i }[-1],
+          tweets: tweets,
+          convert_limit: 30,
+      )
     end
 
-    EMBED_TWEET_HTML = <<~'HTML'
-      <blockquote class="twitter-tweet">
-        <a href="__TWEET_URL__"></a>
-      </blockquote>
-    HTML
+    def chart_data(tweets)
+      data = tweets.each_with_object(Hash.new(0)) do |tweet, memo|
+        t = tweet.created_at
+        min = (t.min.to_f / 5).floor * 5
+        time = Time.new(t.year, t.month, t.day, t.hour, min, 0, '+00:00')
+        memo[time.to_i] += 1
+      end
 
-    EMBED_TWEETS_HTML = <<~'HTML'
-      <html>
-        <head>
-          <style type=text/css>
-            .twitter-tweet {
-              margin: 30px auto 0 auto !important;
-            }
-          </style>
-          <script>
-            window.twttr = (function(d, s, id) {
-              var js, fjs = d.getElementsByTagName(s)[0], t = window.twttr || {};
-              if (d.getElementById(id)) return t;
-              js = d.createElement(s);
-              js.id = id;
-              js.src = "https://platform.twitter.com/widgets.js";
-              fjs.parentNode.insertBefore(js, fjs);
-
-              t._e = [];
-              t.ready = function(f) {
-                  t._e.push(f);
-              };
-
-              return t;
-            }(document, "script", "twitter-wjs"));
-          </script>
-        </head>
-        <body>
-          __TWEETS__
-        </body>
-      </html>
-    HTML
+      data.sort_by { |k, v| k }.map do |timestamp, count|
+        [timestamp * 1000, count]
+      end
+    end
   end
 end

data/lib/twitterscraper/template/tweets.html.erb

@@ -0,0 +1,98 @@
+<html>
+<head>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.27.0/moment.min.js" integrity="sha512-rmZcZsyhe0/MAjquhTgiUcb4d9knaFc7b5xAfju483gbEXTkeJRUMIPk6s3ySZMYUHEcjKbjLjyddGWMrNEvZg==" crossorigin="anonymous"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/moment-timezone/0.5.31/moment-timezone-with-data.min.js" integrity="sha512-HZcf3uHWA+Y2P5KNv+F/xa87/flKVP92kUTe/KXjU8URPshczF1Dx+cL5bw0VBGhmqWAK0UbhcqxBbyiNtAnWQ==" crossorigin="anonymous"></script>
+  <script src="https://code.highcharts.com/stock/highstock.js"></script>
+  <script>
+      function updateTweets() {
+          window.twttr = (function (d, s, id) {
+              var js, fjs = d.getElementsByTagName(s)[0], t = window.twttr || {};
+              if (d.getElementById(id)) return t;
+              js = d.createElement(s);
+              js.id = id;
+              js.src = "https://platform.twitter.com/widgets.js";
+              fjs.parentNode.insertBefore(js, fjs);
+
+              t._e = [];
+              t.ready = function (f) {
+                  t._e.push(f);
+              };
+
+              return t;
+          }(document, "script", "twitter-wjs"));
+      }
+
+      function drawChart() {
+          var data = <%= chart_data %>;
+          Highcharts.setOptions({
+              time: {
+                  timezone: moment.tz.guess()
+              }
+          });
+
+          Highcharts.stockChart('chart', {
+              title: {
+                  text: '<%= tweets.size %> tweets of <%= chart_name %>'
+              },
+              subtitle: {
+                  text: 'since:<%= first_tweet.created_at.localtime.strftime('%Y-%m-%d %H:%M') %> until:<%= last_tweet.created_at.localtime.strftime('%Y-%m-%d %H:%M') %>'
+              },
+              series: [{
+                  data: data
+              }],
+              rangeSelector: {enabled: false},
+              scrollbar: {enabled: false},
+              navigator: {enabled: false},
+              exporting: {enabled: false},
+              credits: {enabled: false}
+          });
+      }
+
+      document.addEventListener("DOMContentLoaded", function () {
+          drawChart();
+          updateTweets();
+      });
+  </script>
+
+  <style type=text/css>
+    .tweets-container {
+      max-width: 550px;
+      margin: 0 auto 0 auto;
+    }
+
+    .twitter-tweet {
+      margin: 15px 0 15px 0 !important;
+    }
+  </style>
+</head>
+<body>
+<div id="chart" style="width: 100vw; height: 400px;"></div>
+
+<div class="tweets-container">
+  <% tweets.each.with_index do |tweet, i| %>
+    <% tweet_time = tweet.created_at.localtime.strftime('%Y-%m-%d %H:%M') %>
+    <% if i < convert_limit %>
+      <blockquote class="twitter-tweet">
+    <% else %>
+      <div class="twitter-tweet" style="border: 1px solid rgb(204, 214, 221);">
+    <% end %>
+
+    <div style="display: grid; grid-template-rows: 24px 24px; grid-template-columns: 48px 1fr;">
+      <div style="grid-row: 1/3; grid-column:  1/2;"><img src="<%= tweet.profile_image_url %>" width="48" height="48" loading="lazy"></div>
+      <div style="grid-row: 1/2; grid-column:  2/3;"><%= tweet.name %></div>
+      <div style="grid-row: 2/3; grid-column:  2/3;"><a href="https://twitter.com/<%= tweet.screen_name %>">@<%= tweet.screen_name %></a></div>
+    </div>
+
+    <div><%= tweet.text %></div>
+    <div><a href="<%= tweet.tweet_url %>"><small><%= tweet_time %></small></a></div>
+
+    <% if i < convert_limit %>
+      </blockquote>
+    <% else %>
+      </div>
+    <% end %>
+  <% end %>
+</div>
+
+</body>
+</html>

data/lib/twitterscraper/tweet.rb

@@ -6,6 +6,7 @@ module Twitterscraper
         :screen_name,
         :name,
         :user_id,
+        :profile_image_url,
         :tweet_id,
         :text,
         :links,
@@ -51,6 +52,11 @@ module Twitterscraper
         end
       end
 
+      # .js-stream-item
+      #   .js-stream-tweet{data: {screen-name:, tweet-id:}}
+      #     .stream-item-header
+      #     .js-tweet-text-container
+      #     .stream-item-footer
       def from_html(text)
         html = Nokogiri::HTML(text)
         from_tweets_html(html.xpath("//li[@class[contains(., 'js-stream-item')]]/div[@class[contains(., 'js-stream-tweet')]]"))
@@ -72,6 +78,8 @@ module Twitterscraper
         end
 
         inner_html = Nokogiri::HTML(html.inner_html)
+
+        profile_image_url = inner_html.xpath("//img[@class[contains(., 'js-action-profile-avatar')]]").first.attr('src').gsub(/_bigger/, '')
         text = inner_html.xpath("//div[@class[contains(., 'js-tweet-text-container')]]/p[@class[contains(., 'js-tweet-text')]]").first.text
         links = inner_html.xpath("//a[@class[contains(., 'twitter-timeline-link')]]").map { |elem| elem.attr('data-expanded-url') }.select { |link| link && !link.include?('pic.twitter') }
         image_urls = inner_html.xpath("//div[@class[contains(., 'AdaptiveMedia-photoContainer')]]").map { |elem| elem.attr('data-image-url') }
@@ -99,6 +107,7 @@ module Twitterscraper
             screen_name: screen_name,
             name: html.attr('data-name'),
             user_id: html.attr('data-user-id').to_i,
+            profile_image_url: profile_image_url,
             tweet_id: tweet_id,
             text: text,
             links: links,

data/lib/twitterscraper/type.rb

@@ -0,0 +1,15 @@
+module Twitterscraper
+  class Type
+    def initialize(value)
+      @value = value
+    end
+
+    def search?
+      @value == 'search'
+    end
+
+    def user?
+      @value == 'user'
+    end
+  end
+end

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Twitterscraper
-  VERSION = '0.14.0'
+  VERSION = '0.17.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twitterscraper-ruby
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.17.0
 platform: ruby
 authors:
 - ts-3156
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-16 00:00:00.000000000 Z
+date: 2020-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -72,7 +72,9 @@ files:
 - lib/twitterscraper/proxy.rb
 - lib/twitterscraper/query.rb
 - lib/twitterscraper/template.rb
+- lib/twitterscraper/template/tweets.html.erb
 - lib/twitterscraper/tweet.rb
+- lib/twitterscraper/type.rb
 - lib/version.rb
 - twitterscraper-ruby.gemspec
 homepage: https://github.com/ts-3156/twitterscraper-ruby