twitterscraper-ruby 0.19.0 → 0.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2056b4a3d9fe7af49429e35b3a1688256fb31b74cabab841a4dd2376a79889d5
-  data.tar.gz: aaaf949da2ba2ae07a0d66e981aebc635c18120de06be705f96c19c92c309911
+  metadata.gz: 73a9e9108284fc79cf5ec6b36b6f7ad3f83f2b4f03a2bc527dc18cb4b33e83c7
+  data.tar.gz: c7fcfdbdd1d808780c56610be9b8717352c812759b9344d9fa87cbd430a8d8e2
 SHA512:
-  metadata.gz: c60824e4c1c0021a3e27451b1708a77bd2e15dd6258fce63ac1b95111d0230c8ab7317bcd76c2faf14d02ebe75ab8d7453924e01eee7d3fcb46eef374f16c575
-  data.tar.gz: 984204bd430b41b76a2d9108df4e778e2bb242010ebd18569bcb662473496826644ba5693db1d475d565bff49a3de7f0eb95fd4c9a3da9e5ed4d6a6219ebb62e
+  metadata.gz: 1019547fe8c37a1bb5b4a9cd96a2737a14491087075ff448b48f72538758337c76ab513e153d4567454b192d30fafaa374913ae0c3548d7802e7bdd478fe4a2f
+  data.tar.gz: 48134e8b6858154850003da8684d3c8b7f124cab6d19e0ce76d05326dc8fef44694b32211e245509993e8b7b1afafa6d95914b05c66b9c95c54bb27d041983fe

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    twitterscraper-ruby (0.19.0)
+    twitterscraper-ruby (0.20.0)
       nokogiri
       parallel
 

data/README.md

@@ -175,7 +175,8 @@ Search operators documentation is in [Standard search operators](https://develop
 | `--limit`      | integer | Stop scraping when *at least* the number of tweets indicated with --limit is scraped. | 100 |
 | `--order`      | string  | Sort a order of the results. | desc(default) or asc |
 | `--threads`    | integer | Set the number of threads twitterscraper-ruby should initiate while scraping for your query. | 2 |
-| `--threads_granularity` | string | | auto |
+| `--threads_granularity` | string | day or hour | auto |
+| `--chart_grouping` | string | day, hour or minute | auto |
 | `--proxy`      | boolean | Scrape https://twitter.com/search via proxies. | true(default) or false |
 | `--cache`      | boolean | Enable caching. | true(default) or false |
 | `--format`     | string  | The format of the output. | json(default) or html |

data/lib/twitterscraper/cli.rb

@@ -74,6 +74,7 @@ module Twitterscraper
           'order:',
           'threads:',
           'threads_granularity:',
+          'chart_grouping:',
           'output:',
           'format:',
           'cache:',

data/lib/twitterscraper/query.rb

@@ -131,19 +131,27 @@ module Twitterscraper
       if start_date && end_date
         if start_date == end_date
           raise Error.new('Please specify different values for :start_date and :end_date.')
-        elsif start_date > end_date
+        elsif Date.parse(start_date) > Date.parse(end_date)
           raise Error.new(':start_date must occur before :end_date.')
         end
       end
 
       if start_date
-        if start_date < OLDEST_DATE
+        if Date.parse(start_date) < OLDEST_DATE
           raise Error.new(":start_date must be greater than or equal to #{OLDEST_DATE}")
         end
       end
     end
 
-    def build_queries(query, start_date, end_date, threads_granularity)
+    def build_queries(query, start_date, end_date, threads_granularity, type)
+      if type.search?
+        start_date = Date.parse(start_date) if start_date.is_a?(String)
+        end_date = Date.parse(end_date) if end_date.is_a?(String)
+      elsif type.user?
+        start_date = nil
+        end_date = nil
+      end
+
       if start_date && end_date
         if threads_granularity == 'auto'
           threads_granularity = start_date.upto(end_date - 1).to_a.size >= 28 ? 'day' : 'hour'
@@ -151,7 +159,7 @@ module Twitterscraper
 
         if threads_granularity == 'day'
           date_range = start_date.upto(end_date - 1)
-          queries = date_range.map { |date| query + " since:#{date} until:#{date + 1}" }
+          queries = date_range.map { |date| query + " since:#{date}_00:00:00_UTC until:#{date + 1}_00:00:00_UTC" }
         elsif threads_granularity == 'hour'
           time = Time.utc(start_date.year, start_date.month, start_date.day, 0, 0, 0)
           end_time = Time.utc(end_date.year, end_date.month, end_date.day, 0, 0, 0)
@@ -159,19 +167,21 @@ module Twitterscraper
 
           while true
             if time < Time.now.utc
-              queries << (query + " since:#{time.strftime('%Y-%m-%d_%H:00:00')}_UTC until:#{(time + 3600).strftime('%Y-%m-%d_%H:00:00')}_UTC")
+              queries << (query + " since:#{time.strftime('%Y-%m-%d_%H')}:00:00_UTC until:#{(time + 3600).strftime('%Y-%m-%d_%H')}:00:00_UTC")
             end
             time += 3600
             break if time >= end_time
           end
+        else
+          raise Error.new("Invalid :threads_granularity value=#{threads_granularity}")
         end
 
         @queries = queries
 
       elsif start_date
-        [query + " since:#{start_date}"]
+        [query + " since:#{start_date}_00:00:00_UTC"]
       elsif end_date
-        [query + " until:#{end_date}"]
+        [query + " until:#{end_date}_00:00:00_UTC"]
       else
         [query]
       end
@@ -214,15 +224,7 @@ module Twitterscraper
 
     def query_tweets(query, type: 'search', start_date: nil, end_date: nil, lang: nil, limit: 100, daily_limit: nil, order: 'desc', threads: 10, threads_granularity: 'auto')
       type = Type.new(type)
-      if type.search?
-        start_date = Date.parse(start_date) if start_date && start_date.is_a?(String)
-        end_date = Date.parse(end_date) if end_date && end_date.is_a?(String)
-      elsif type.user?
-        start_date = nil
-        end_date = nil
-      end
-
-      queries = build_queries(query, start_date, end_date, threads_granularity)
+      queries = build_queries(query, start_date, end_date, threads_granularity, type)
       if threads > queries.size
         threads = queries.size
       end

data/lib/twitterscraper/template.rb

@@ -5,10 +5,11 @@ module Twitterscraper
       template = ERB.new(File.read(path))
 
       tweets = tweets.sort_by { |t| t.created_at.to_i }
+      grouping = options['chart_grouping'] || 'auto'
 
       template.result_with_hash(
           chart_name: name,
-          chart_data: chart_data(tweets).to_json,
+          chart_data: chart_data(tweets, grouping: grouping).to_json,
           first_tweet: tweets[0],
           last_tweet: tweets[-1],
           tweets: tweets,

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Twitterscraper
-  VERSION = '0.19.0'
+  VERSION = '0.20.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twitterscraper-ruby
 version: !ruby/object:Gem::Version
-  version: 0.19.0
+  version: 0.20.0
 platform: ruby
 authors:
 - ts-3156
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-23 00:00:00.000000000 Z
+date: 2020-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri