twitterscraper-ruby 0.15.2 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f7d320841125d9a582ece6083f421f0abf301addbc5c5c2a3d2b2c09bedbc33
-  data.tar.gz: 6ea43165ffa4f37c4319566689a42f2f275d8a70402b0d6b4164df519fee90b5
+  metadata.gz: 66dda5275a9067d328f6637f127895ded954534d304e5e4b349f286a271a08d8
+  data.tar.gz: 6c3ffb3fba82376fc2de49514245ea96c7cb4fa16c32dcd2fff1ab1ae327bd14
 SHA512:
-  metadata.gz: ee3756538ec28e9f0113e611e2731ec33107dabacf7cb730b257d6c94351407ef171a9bc91402a589fa73fdb6b705f73b11582766af1d04a3413b8bc79dc6619
-  data.tar.gz: 78200dc658a9c1cf43ed7367e499b0d1b243728aecb2ffd7366b5612f8905bb33d27ab7e1412327d05b7fff159196fe9e24d18c8cc4c24898af10533fbdf43df
+  metadata.gz: 24267284f4f29adc86d5bbe70a30bbe31d6d898546576065f1a9accafc3944a352117bbf6eb0de273743a00fb2d26c5cf37ed016cc0324187a25ca279230d812
+  data.tar.gz: 0bc9f01659560c83b0289bf63119849135b7ec27520dd03c7abd645da99ef660ca4b5fd12301b359cd5cc45a82914d7ceae88ad93ad756fde166718b3d0fe6c2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    twitterscraper-ruby (0.15.2)
+    twitterscraper-ruby (0.16.0)
       nokogiri
       parallel
 

data/lib/twitterscraper/cache.rb

@@ -4,7 +4,7 @@ require 'digest/md5'
 module Twitterscraper
   class Cache
     def initialize()
-      @ttl = 3600 # 1 hour
+      @ttl = 86400 # 1 day
       @dir = 'cache'
       Dir.mkdir(@dir) unless File.exist?(@dir)
     end
@@ -25,6 +25,12 @@ module Twitterscraper
       File.write(file, entry.to_json)
     end
 
+    def delete(key)
+      key = cache_key(key)
+      file = File.join(@dir, key)
+      File.delete(file) if File.exist?(file)
+    end
+
     def fetch(key, &block)
       if (value = read(key))
         value

data/lib/twitterscraper/cli.rb

@@ -27,16 +27,16 @@ module Twitterscraper
       }
       client = Twitterscraper::Client.new(cache: options['cache'], proxy: options['proxy'])
       tweets = client.query_tweets(options['query'], query_options)
-      export(tweets) unless tweets.empty?
+      export(options['query'], tweets) unless tweets.empty?
     end
 
-    def export(tweets)
+    def export(name, tweets)
       write_json = lambda { File.write(options['output'], generate_json(tweets)) }
 
       if options['format'] == 'json'
         write_json.call
       elsif options['format'] == 'html'
-        File.write('tweets.html', Template.tweets_embedded_html(tweets))
+        File.write(options['output'], Template.new.tweets_embedded_html(name, tweets, options))
       else
         write_json.call
       end

data/lib/twitterscraper/query.rb

@@ -69,7 +69,6 @@ module Twitterscraper
       else
         json_resp = JSON.parse(text)
         items_html = json_resp['items_html'] || ''
-        logger.warn json_resp['message'] if json_resp['message'] # Sorry, you are rate limited.
       end
 
       [items_html, json_resp]
@@ -100,6 +99,12 @@ module Twitterscraper
 
       html, json_resp = parse_single_page(response, pos.nil?)
 
+      if json_resp && json_resp['message']
+        logger.warn json_resp['message'] # Sorry, you are rate limited.
+        @stop_requested = true
+        Cache.new.delete(url) if cache_enabled?
+      end
+
       tweets = Tweet.from_html(html)
 
       if tweets.empty?
@@ -140,19 +145,27 @@ module Twitterscraper
           raise Error.new(":start_date must be greater than or equal to #{OLDEST_DATE}")
         end
       end
-
-      if end_date
-        today = Date.today
-        if end_date > Date.today
-          raise Error.new(":end_date must be less than or equal to today(#{today})")
-        end
-      end
     end
 
     def build_queries(query, start_date, end_date)
       if start_date && end_date
-        date_range = start_date.upto(end_date - 1)
-        date_range.map { |date| query + " since:#{date} until:#{date + 1}" }
+        # date_range = start_date.upto(end_date - 1)
+        # date_range.map { |date| query + " since:#{date} until:#{date + 1}" }
+
+        queries = []
+        time = Time.utc(start_date.year, start_date.month, start_date.day, 0, 0, 0)
+        end_time = Time.utc(end_date.year, end_date.month, end_date.day, 0, 0, 0)
+
+        while true
+          if time < Time.now.utc
+            queries << (query + " since:#{time.strftime('%Y-%m-%d_%H:00:00')}_UTC until:#{(time + 3600).strftime('%Y-%m-%d_%H:00:00')}_UTC")
+          end
+          time += 3600
+          break if time >= end_time
+        end
+
+        queries
+
       elsif start_date
         [query + " since:#{start_date}"]
       elsif end_date
@@ -202,7 +215,6 @@ module Twitterscraper
       queries = build_queries(query, start_date, end_date)
       type = Type.new(type)
       if threads > queries.size
-        logger.warn 'The maximum number of :threads is the number of dates between :start_date and :end_date.'
         threads = queries.size
       end
       if proxy_enabled?
@@ -214,7 +226,6 @@ module Twitterscraper
       end
       logger.debug "Cache #{cache_enabled? ? 'enabled' : 'disabled'}"
 
-
       validate_options!(queries, type: type, start_date: start_date, end_date: end_date, lang: lang, limit: limit, threads: threads)
 
       logger.info "The number of threads #{threads}"

data/lib/twitterscraper/template.rb

@@ -1,48 +1,30 @@
 module Twitterscraper
-  module Template
-    module_function
+  class Template
+    def tweets_embedded_html(name, tweets, options)
+      path = File.join(File.dirname(__FILE__), 'template/tweets.html.erb')
+      template = ERB.new(File.read(path))
 
-    def tweets_embedded_html(tweets)
-      tweets_html = tweets.map { |t| EMBED_TWEET_HTML.sub('__TWEET_URL__', t.tweet_url) }
-      EMBED_TWEETS_HTML.sub('__TWEETS__', tweets_html.join)
+      template.result_with_hash(
+          chart_name: name,
+          chart_data: chart_data(tweets).to_json,
+          first_tweet: tweets.sort_by { |t| t.created_at.to_i }[0],
+          last_tweet: tweets.sort_by { |t| t.created_at.to_i }[-1],
+          tweets_size: tweets.size,
+          tweets: tweets.take(50)
+      )
     end
 
-    EMBED_TWEET_HTML = <<~'HTML'
-      <blockquote class="twitter-tweet">
-        <a href="__TWEET_URL__"></a>
-      </blockquote>
-    HTML
+    def chart_data(tweets)
+      data = tweets.each_with_object(Hash.new(0)) do |tweet, memo|
+        t = tweet.created_at
+        min = (t.min.to_f / 5).floor * 5
+        time = Time.new(t.year, t.month, t.day, t.hour, min, 0, '+00:00')
+        memo[time.to_i] += 1
+      end
 
-    EMBED_TWEETS_HTML = <<~'HTML'
-      <html>
-        <head>
-          <style type=text/css>
-            .twitter-tweet {
-              margin: 30px auto 0 auto !important;
-            }
-          </style>
-          <script>
-            window.twttr = (function(d, s, id) {
-              var js, fjs = d.getElementsByTagName(s)[0], t = window.twttr || {};
-              if (d.getElementById(id)) return t;
-              js = d.createElement(s);
-              js.id = id;
-              js.src = "https://platform.twitter.com/widgets.js";
-              fjs.parentNode.insertBefore(js, fjs);
-
-              t._e = [];
-              t.ready = function(f) {
-                  t._e.push(f);
-              };
-
-              return t;
-            }(document, "script", "twitter-wjs"));
-          </script>
-        </head>
-        <body>
-          __TWEETS__
-        </body>
-      </html>
-    HTML
+      data.sort_by { |k, v| k }.map do |timestamp, count|
+        [timestamp * 1000, count]
+      end
+    end
   end
 end

data/lib/twitterscraper/template/tweets.html.erb

@@ -0,0 +1,82 @@
+<html>
+<head>
+  <script>
+      window.twttr = (function (d, s, id) {
+          var js, fjs = d.getElementsByTagName(s)[0], t = window.twttr || {};
+          if (d.getElementById(id)) return t;
+          js = d.createElement(s);
+          js.id = id;
+          js.src = "https://platform.twitter.com/widgets.js";
+          fjs.parentNode.insertBefore(js, fjs);
+
+          t._e = [];
+          t.ready = function (f) {
+              t._e.push(f);
+          };
+
+          return t;
+      }(document, "script", "twitter-wjs"));
+  </script>
+
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.27.0/moment.min.js" integrity="sha512-rmZcZsyhe0/MAjquhTgiUcb4d9knaFc7b5xAfju483gbEXTkeJRUMIPk6s3ySZMYUHEcjKbjLjyddGWMrNEvZg==" crossorigin="anonymous"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/moment-timezone/0.5.31/moment-timezone-with-data.min.js" integrity="sha512-HZcf3uHWA+Y2P5KNv+F/xa87/flKVP92kUTe/KXjU8URPshczF1Dx+cL5bw0VBGhmqWAK0UbhcqxBbyiNtAnWQ==" crossorigin="anonymous"></script>
+  <script src="https://code.highcharts.com/stock/highstock.js"></script>
+  <script>
+      function drawChart() {
+          Highcharts.setOptions({
+              time: {
+                  timezone: moment.tz.guess()
+              }
+          });
+
+          Highcharts.stockChart('chart', {
+              title: {
+                  text: '<%= tweets_size %> tweets of <%= chart_name %>'
+              },
+              subtitle: {
+                  text: 'since:<%= first_tweet.created_at.localtime %> until:<%= last_tweet.created_at.localtime %>'
+              },
+              series: [{
+                  data: <%= chart_data %>
+              }],
+              rangeSelector: {enabled: false},
+              scrollbar: {enabled: false},
+              navigator: {enabled: false},
+              exporting: {enabled: false},
+              credits: {enabled: false}
+          });
+      }
+
+      document.addEventListener("DOMContentLoaded", function () {
+          drawChart();
+      });
+  </script>
+
+  <style type=text/css>
+    .tweets-container {
+      max-width: 550px;
+      margin: 0 auto 0 auto;
+    }
+
+    .twitter-tweet {
+      margin: 15px 0 15px 0 !important;
+    }
+  </style>
+</head>
+<body>
+<div id="chart"></div>
+
+<div class="tweets-container">
+  <% tweets.each do |tweet| %>
+    <blockquote class="twitter-tweet">
+      <a href="<%= tweet.tweet_url %>"></a>
+    </blockquote>
+  <% end %>
+
+  <% if tweets_size > tweets.size %>
+    <div>and more!</div>
+  <% end %>
+</div>
+
+</body>
+</html>

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Twitterscraper
-  VERSION = '0.15.2'
+  VERSION = '0.16.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twitterscraper-ruby
 version: !ruby/object:Gem::Version
-  version: 0.15.2
+  version: 0.16.0
 platform: ruby
 authors:
 - ts-3156
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-17 00:00:00.000000000 Z
+date: 2020-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -72,6 +72,7 @@ files:
 - lib/twitterscraper/proxy.rb
 - lib/twitterscraper/query.rb
 - lib/twitterscraper/template.rb
+- lib/twitterscraper/template/tweets.html.erb
 - lib/twitterscraper/tweet.rb
 - lib/twitterscraper/type.rb
 - lib/version.rb