twitterscraper-ruby 0.12.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e98afb0444b724e0c9c29f6b888c017166859d1252337f34686526060ca8368d
-  data.tar.gz: 5ef3ff7f86d9a0c9dd1883d55498049d0f164aa7c71a7c9c2bbf0a89ae9bb32c
+  metadata.gz: bafdfd47b386ef7f717dc5846102c8a5153f4660e61d3559f6834cdca340c19c
+  data.tar.gz: fb5564629d89ae83c916d868e9fd401fdca1b423fbeb2d6945b0831c0d8ecf11
 SHA512:
-  metadata.gz: 04ef61c57545cbbdbbe5da53d1f24cf064b7d1c61ad3da9bc57a361d24ed24480c4f68fa1fea67345ceff4d4d4685f046a4586f55ebe8f3dc0ca6332c7c2d928
-  data.tar.gz: f5fd19c8289c7caf574dc78f754ba9aaf9446f3819b394d14414909b1505e0f9b25181802448d28285de8db81a27e12e0e65d1e1a0b2b0e5df8e7e73d6263e14
+  metadata.gz: 5e9c819b318a908c73f56de0a638c7c819cc1e31c867812ec9ffa3e23362318db3dfc1fe5ffde53c4769bffdf1f62efdb4701bf9b1efe874625cdb6ce21ef1bc
+  data.tar.gz: aa75f3a328f6c2c278738962e7d6e9ea747841343362e8a0f226fd76b316b6ab05e63c93e495ae39009fd3f0a1c4eb0657bc66e1b22416e6a695cc34b4059643

data/.rspec

@@ -0,0 +1,2 @@
+-fd
+--require spec_helper

data/Gemfile

@@ -5,3 +5,4 @@ gemspec
 
 gem "rake", "~> 12.0"
 gem "minitest", "~> 5.0"
+gem "rspec"

data/Gemfile.lock

@@ -1,19 +1,33 @@
 PATH
   remote: .
   specs:
-    twitterscraper-ruby (0.12.0)
+    twitterscraper-ruby (0.13.0)
       nokogiri
       parallel
 
 GEM
   remote: https://rubygems.org/
   specs:
+    diff-lcs (1.4.4)
     mini_portile2 (2.4.0)
     minitest (5.14.1)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     parallel (1.19.2)
     rake (12.3.3)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
 
 PLATFORMS
   ruby
@@ -21,6 +35,7 @@ PLATFORMS
 DEPENDENCIES
   minitest (~> 5.0)
   rake (~> 12.0)
+  rspec
   twitterscraper-ruby!
 
 BUNDLED WITH

data/lib/twitterscraper/cli.rb

@@ -22,9 +22,8 @@ module Twitterscraper
           limit: options['limit'],
           daily_limit: options['daily_limit'],
           threads: options['threads'],
-          proxy: options['proxy']
       }
-      client = Twitterscraper::Client.new(cache: options['cache'])
+      client = Twitterscraper::Client.new(cache: options['cache'], proxy: options['proxy'])
       tweets = client.query_tweets(options['query'], query_options)
       export(tweets) unless tweets.empty?
     end
@@ -68,8 +67,8 @@ module Twitterscraper
           'threads:',
           'output:',
           'format:',
-          'cache',
-          'proxy',
+          'cache:',
+          'proxy:',
           'pretty',
           'verbose',
       )
@@ -82,6 +81,9 @@ module Twitterscraper
       options['format'] ||= 'json'
       options['output'] ||= "tweets.#{options['format']}"
 
+      options['cache'] = options['cache'] != 'false'
+      options['proxy'] = options['proxy'] != 'false'
+
       options
     end
 

data/lib/twitterscraper/client.rb

@@ -2,12 +2,17 @@ module Twitterscraper
   class Client
     include Query
 
-    def initialize(cache: false)
+    def initialize(cache: true, proxy: true)
       @cache = cache
+      @proxy = proxy
     end
 
     def cache_enabled?
       @cache
     end
+
+    def proxy_enabled?
+      @proxy
+    end
   end
 end

data/lib/twitterscraper/query.rb

@@ -116,7 +116,8 @@ module Twitterscraper
 
     OLDEST_DATE = Date.parse('2006-03-21')
 
-    def validate_options!(query, start_date:, end_date:, lang:, limit:, threads:, proxy:)
+    def validate_options!(queries, start_date:, end_date:, lang:, limit:, threads:)
+      query = queries[0]
       if query.nil? || query == ''
         raise Error.new('Please specify a search query.')
       end
@@ -194,16 +195,26 @@ module Twitterscraper
       @stop_requested
     end
 
-    def query_tweets(query, start_date: nil, end_date: nil, lang: '', limit: 100, daily_limit: nil, threads: 2, proxy: false)
+    def query_tweets(query, start_date: nil, end_date: nil, lang: '', limit: 100, daily_limit: nil, threads: 2)
       start_date = Date.parse(start_date) if start_date && start_date.is_a?(String)
       end_date = Date.parse(end_date) if end_date && end_date.is_a?(String)
       queries = build_queries(query, start_date, end_date)
-      threads = queries.size if threads > queries.size
-      proxies = proxy ? Proxy::Pool.new : []
+      if threads > queries.size
+        logger.warn 'The maximum number of :threads is the number of dates between :start_date and :end_date.'
+        threads = queries.size
+      end
+      if proxy_enabled?
+        proxies = Proxy::Pool.new
+        logger.debug "Fetch #{proxies.size} proxies"
+      else
+        proxies = []
+        logger.debug 'Proxy disabled'
+      end
+      logger.debug "Cache #{cache_enabled? ? 'enabled' : 'disabled'}"
+
 
-      validate_options!(queries[0], start_date: start_date, end_date: end_date, lang: lang, limit: limit, threads: threads, proxy: proxy)
+      validate_options!(queries, start_date: start_date, end_date: end_date, lang: lang, limit: limit, threads: threads)
 
-      logger.debug "Fetch #{proxies.size} proxies" if proxy
       logger.info "The number of threads #{threads}"
 
       headers = {'User-Agent': USER_AGENT_LIST.sample, 'X-Requested-With': 'XMLHttpRequest'}

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Twitterscraper
-  VERSION = '0.12.0'
+  VERSION = '0.13.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twitterscraper-ruby
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.13.0
 platform: ruby
 authors:
 - ts-3156
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-15 00:00:00.000000000 Z
+date: 2020-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -48,6 +48,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".irbrc"
+- ".rspec"
 - ".ruby-version"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md