twitterscraper-ruby 0.17.0 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ac0c10b18d836983cc6b73e25b9ed333af2f620106a07c6bc6a40058fb127895
-  data.tar.gz: e6fc18219d9127fb30ba57e39dc4656c0f0a3c108428d959de5bac9e7d317088
+  metadata.gz: 8e9bdefe1c4d10e6d9f1d12aeb279b2a3751c570e96e05daaf849dd423bb03bf
+  data.tar.gz: 7de97de19daeecce2837fe8e5999b6c9490ab49a18a2ab9e603bf4d039abc4b9
 SHA512:
-  metadata.gz: 90cbf06b606878dc36b4bba44669139c273bf03b08a777ad87036834841bcb4b052e0559813dc56e4be124442abfc5a7fc44c5c9524c74929ca02b1d287d346b
-  data.tar.gz: ada0b74ee42ff62964b73ad9b49358227cdaf4fc87420cf12cf65af95168ad9775615a504345ebc83d3b791e9c0d892691c55bc477eddd647b3e8934f752fb9c
+  metadata.gz: 55b7e0b52b2ce44418305798ed27a677405244a48f5ad0a797e3abf7958b0581a313ebd33f3f69b891ba7454f8f5c9c0db845c9ca8be321cd27212932821776e
+  data.tar.gz: 8fe97a0dc164fc0108b8e6a35843fba19ade5fbaf4f1ee2b4a400afbd3bdbb220a49dfbef4fceb1d8ecc43df3b4f4b7bad0ee5ea94c0aac464c0477e42efb866

data/.gitignore

@@ -8,3 +8,4 @@
 /tmp/
 /cache
 /.idea
+.DS_Store

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    twitterscraper-ruby (0.17.0)
+    twitterscraper-ruby (0.18.0)
       nokogiri
       parallel
 

data/lib/twitterscraper/cli.rb

@@ -32,12 +32,14 @@ module Twitterscraper
     end
 
     def export(name, tweets)
-      write_json = lambda { File.write(options['output'], generate_json(tweets)) }
+      filepath = options['output']
+      Dir.mkdir(File.dirname(filepath)) unless File.exist?(File.dirname(filepath))
+      write_json = lambda { File.write(filepath, generate_json(tweets)) }
 
       if options['format'] == 'json'
         write_json.call
       elsif options['format'] == 'html'
-        File.write(options['output'], Template.new.tweets_embedded_html(name, tweets, options))
+        File.write(filepath, Template.new.tweets_embedded_html(name, tweets, options))
       else
         write_json.call
       end
@@ -97,9 +99,9 @@ module Twitterscraper
     end
 
     def build_output_name(options)
-      query = ERB::Util.url_encode(options['query'])
+      query = options['query'].gsub(/[ :?#&]/, '_')
       date = [options['start_date'], options['end_date']].select { |val| val && !val.empty? }.join('_')
-      [options['type'], 'tweets', date, query].compact.join('_') + '.' + options['format']
+      File.join('out', [options['type'], 'tweets', date, query].compact.join('_') + '.' + options['format'])
     end
 
     def initialize_logger

data/lib/twitterscraper/query.rb

@@ -76,9 +76,9 @@ module Twitterscraper
 
     def query_single_page(query, lang, type, pos, headers: [], proxies: [])
       logger.info "Querying #{query}"
-      query = ERB::Util.url_encode(query)
+      encoded_query = ERB::Util.url_encode(query)
 
-      url = build_query_url(query, lang, type, pos)
+      url = build_query_url(encoded_query, lang, type, pos)
       http_request = lambda do
         logger.debug "Scraping tweets from url=#{url}"
         get_single_page(url, headers, proxies)
@@ -92,6 +92,10 @@ module Twitterscraper
           response = http_request.call
           client.write(url, response) unless stop_requested?
         end
+        if @queries && query == @queries.last && pos.nil?
+          logger.debug "Delete a cache query=#{query}"
+          client.delete(url)
+        end
       else
         response = http_request.call
       end
@@ -170,7 +174,7 @@ module Twitterscraper
           end
         end
 
-        queries
+        @queries = queries
 
       elsif start_date
         [query + " since:#{start_date}"]

data/lib/twitterscraper/template.rb

@@ -4,25 +4,54 @@ module Twitterscraper
       path = File.join(File.dirname(__FILE__), 'template/tweets.html.erb')
       template = ERB.new(File.read(path))
 
+      tweets = tweets.sort_by { |t| t.created_at.to_i }
+
       template.result_with_hash(
           chart_name: name,
           chart_data: chart_data(tweets).to_json,
-          first_tweet: tweets.sort_by { |t| t.created_at.to_i }[0],
-          last_tweet: tweets.sort_by { |t| t.created_at.to_i }[-1],
+          first_tweet: tweets[0],
+          last_tweet: tweets[-1],
           tweets: tweets,
           convert_limit: 30,
       )
     end
 
-    def chart_data(tweets)
+    def chart_data(tweets, trimming: true, smoothing: true)
+      min_interval = 5
+
       data = tweets.each_with_object(Hash.new(0)) do |tweet, memo|
         t = tweet.created_at
-        min = (t.min.to_f / 5).floor * 5
+        min = (t.min.to_f / min_interval).floor * min_interval
         time = Time.new(t.year, t.month, t.day, t.hour, min, 0, '+00:00')
         memo[time.to_i] += 1
       end
 
-      data.sort_by { |k, v| k }.map do |timestamp, count|
+      if false && trimming
+        data.keys.sort.each.with_index do |timestamp, i|
+          break if data.size - 1 == i
+          if data[i] == 0 && data[i + 1] == 0
+            data.delete(timestamp)
+          end
+        end
+      end
+
+      if false && smoothing
+        time = data.keys.min
+        max_time = data.keys.max
+        sec_interval = 60 * min_interval
+
+        while true
+          next_time = time + sec_interval
+          break if next_time + sec_interval > max_time
+
+          unless data.has_key?(next_time)
+            data[next_time] = (data[time] + data[next_time + sec_interval]) / 2
+          end
+          time = next_time
+        end
+      end
+
+      data.sort_by { |k, _| k }.map do |timestamp, count|
         [timestamp * 1000, count]
       end
     end

data/lib/twitterscraper/template/tweets.html.erb

@@ -23,14 +23,14 @@
       }
 
       function drawChart() {
-          var data = <%= chart_data %>;
           Highcharts.setOptions({
               time: {
                   timezone: moment.tz.guess()
               }
           });
 
-          Highcharts.stockChart('chart', {
+          var data = <%= chart_data %>;
+          var config = {
               title: {
                   text: '<%= tweets.size %> tweets of <%= chart_name %>'
               },
@@ -45,7 +45,9 @@
               navigator: {enabled: false},
               exporting: {enabled: false},
               credits: {enabled: false}
-          });
+          };
+
+          Highcharts.stockChart('chart-container', config);
       }
 
       document.addEventListener("DOMContentLoaded", function () {
@@ -55,6 +57,15 @@
   </script>
 
   <style type=text/css>
+    #chart-container {
+      max-width: 1200px;
+      height: 675px;
+      margin: 0 auto;
+      border: 1px solid rgb(204, 214, 221);
+      display: flex;
+      justify-content: center;
+      align-items: center;
+    }
     .tweets-container {
       max-width: 550px;
       margin: 0 auto 0 auto;
@@ -66,10 +77,10 @@
   </style>
 </head>
 <body>
-<div id="chart" style="width: 100vw; height: 400px;"></div>
+<div id="chart-container"><div style="color: gray;">Loading...</div></div>
 
 <div class="tweets-container">
-  <% tweets.each.with_index do |tweet, i| %>
+  <% tweets.sort_by { |t| -t.created_at.to_i }.each.with_index do |tweet, i| %>
     <% tweet_time = tweet.created_at.localtime.strftime('%Y-%m-%d %H:%M') %>
     <% if i < convert_limit %>
       <blockquote class="twitter-tweet">

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Twitterscraper
-  VERSION = '0.17.0'
+  VERSION = '0.18.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twitterscraper-ruby
 version: !ruby/object:Gem::Version
-  version: 0.17.0
+  version: 0.18.0
 platform: ruby
 authors:
 - ts-3156
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-18 00:00:00.000000000 Z
+date: 2020-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri