twitterscraper-ruby 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6701ff59f3eb13db9e3b2d024ea983264b528194d64f4f03a95f3576338ed77
-  data.tar.gz: 74106816dd406ef1b355b4d4fc94b1baf4509465f6d5bf1ea8f7c654e518eec0
+  metadata.gz: fe6db831d59218f3e701e0211487d79ef2354524610f8338a1a17e4cc426e437
+  data.tar.gz: 34cd890b8d2837bcacb3ab7b03fb43845294eecb9d7b0c4891c048eacedbe233
 SHA512:
-  metadata.gz: 9710fb74c90dcbc17a22dd613cfe4dce75106951f1e55cd9cfa94a825ecf0b6773a2851ff1cca842f83b5207d3744ac63bcce29031061b0a0ac84cc12d62b8a3
-  data.tar.gz: f0e7cd90ecb773a1837be9245b83f51d60f25d48cab716e3584a8d3e1b6f0fe4951eadaf034850599205209d2bf8d7cd4ebfda97f9f862a528c393a2f81887a7
+  metadata.gz: 6c16c89ca290cc3c9ed5fd245c5aa26e5386c95011cfa14277e774e860359495cafec1624fba0af55de98ebbb34abb599e75e210bdbb18b3b11e49bc1527b643
+  data.tar.gz: d54e25e0294eddf8226c0e27a1d46c6128e9066c9d04edc429b382498c0de1af7ccf2e5c1333ad2031210bbefd7f9b7edc73b9a0e79ab2bd3673674b2e648f3c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    twitterscraper-ruby (0.5.0)
+    twitterscraper-ruby (0.6.0)
       nokogiri
       parallel
 

data/lib/twitterscraper.rb

@@ -9,10 +9,9 @@ require 'version'
 
 module Twitterscraper
   class Error < StandardError; end
-  # Your code goes here...
 
   def self.logger
-    @logger ||= ::Logger.new(STDOUT)
+    @logger ||= ::Logger.new(STDOUT, level: ::Logger::INFO)
   end
 
   def self.logger=(logger)

data/lib/twitterscraper/cli.rb

@@ -8,18 +8,24 @@ module Twitterscraper
   class Cli
     def parse
       @options = parse_options(ARGV)
+      initialize_logger
     end
 
     def run
-      client = Twitterscraper::Client.new
-      limit = options['limit'] ? options['limit'].to_i : 100
-      threads = options['threads'] ? options['threads'].to_i : 2
-      tweets = client.query_tweets(options['query'], limit: limit, threads: threads, start_date: options['start_date'], end_date: options['end_date'])
-      File.write('tweets.json', generate_json(tweets))
-    end
+      print_help || return if print_help?
+      print_version || return if print_version?
 
-    def options
-      @options
+      client = Twitterscraper::Client.new
+      query_options = {
+          start_date: options['start_date'],
+          end_date: options['end_date'],
+          lang: options['lang'],
+          limit: options['limit'],
+          threads: options['threads'],
+          proxy: options['proxy']
+      }
+      tweets = client.query_tweets(options['query'], query_options)
+      File.write(options['output'], generate_json(tweets))
     end
 
     def generate_json(tweets)
@@ -30,16 +36,57 @@ module Twitterscraper
       end
     end
 
+    def options
+      @options
+    end
+
     def parse_options(argv)
-      argv.getopts(
+      options = argv.getopts(
           'h',
+          'help',
+          'v',
+          'version',
           'query:',
-          'limit:',
           'start_date:',
           'end_date:',
+          'lang:',
+          'limit:',
           'threads:',
+          'output:',
+          'proxy',
           'pretty',
+          'verbose',
       )
+
+      options['lang'] ||= ''
+      options['limit'] = (options['limit'] || 100).to_i
+      options['threads'] = (options['threads'] || 2).to_i
+      options['output'] ||= 'tweets.json'
+
+      options
+    end
+
+    def initialize_logger
+      Twitterscraper.logger.level = ::Logger::DEBUG if options['verbose']
+    end
+
+    def print_help?
+      options['h'] || options['help']
+    end
+
+    def print_help
+      puts <<~'SHELL'
+        Usage:
+          twitterscraper --query KEYWORD --limit 100 --threads 10 --start_date 2020-07-01 --end_date 2020-07-10 --lang ja --proxy --output output.json
+      SHELL
+    end
+
+    def print_version?
+      options['v'] || options['version']
+    end
+
+    def print_version
+      puts "twitterscraper-#{Twitterscraper::VERSION}"
     end
   end
 end

data/lib/twitterscraper/proxy.rb

@@ -6,9 +6,9 @@ module Twitterscraper
     class RetryExhausted < StandardError
     end
 
-    class Result
-      def initialize(items)
-        @items = items
+    class Pool
+      def initialize
+        @items = Proxy.get_proxies
         @cur_index = 0
       end
 
@@ -31,7 +31,6 @@ module Twitterscraper
       def reload
         @items = Proxy.get_proxies
         @cur_index = 0
-        Twitterscraper.logger.debug "Reload #{proxies.size} proxies"
       end
     end
 
@@ -46,13 +45,14 @@ module Twitterscraper
 
       table.xpath('tbody/tr').each do |tr|
         cells = tr.xpath('td')
-        ip, port, https = [0, 1, 6].map { |i| cells[i].text.strip }
+        ip, port, anonymity, https = [0, 1, 4, 6].map { |i| cells[i].text.strip }
+        next unless ['elite proxy', 'anonymous'].include?(anonymity)
         next if https == 'no'
         proxies << ip + ':' + port
       end
 
       Twitterscraper.logger.debug "Fetch #{proxies.size} proxies"
-      Result.new(proxies.shuffle)
+      proxies.shuffle
     rescue => e
       if (retries -= 1) > 0
         retry

data/lib/twitterscraper/query.rb

@@ -91,7 +91,7 @@ module Twitterscraper
       end
     end
 
-    def query_tweets(query, start_date: nil, end_date: nil, limit: 100, threads: 2, lang: '')
+    def query_tweets(query, start_date: nil, end_date: nil, lang: '', limit: 100, threads: 2, proxy: false)
       start_date = start_date ? Date.parse(start_date) : Date.parse('2006-3-21')
       end_date = end_date ? Date.parse(end_date) : Date.today
       if start_date == end_date
@@ -100,19 +100,20 @@ module Twitterscraper
         raise ':start_date must occur before :end_date.'
       end
 
-      proxies = Twitterscraper::Proxy.get_proxies
+      proxies = proxy ? Twitterscraper::Proxy::Pool.new : []
 
       date_range = start_date.upto(end_date - 1)
       queries = date_range.map { |date| query + " since:#{date} until:#{date + 1}" }
       threads = queries.size if threads > queries.size
       logger.info("Threads #{threads}")
 
+      headers = {'User-Agent': USER_AGENT_LIST.sample, 'X-Requested-With': 'XMLHttpRequest'}
+      logger.info("Headers #{headers}")
+
       all_tweets = []
       mutex = Mutex.new
 
       Parallel.each(queries, in_threads: threads) do |query|
-        headers = {'User-Agent': USER_AGENT_LIST.sample, 'X-Requested-With': 'XMLHttpRequest'}
-        logger.info("Headers #{headers}")
 
         pos = nil
 

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Twitterscraper
-  VERSION = '0.5.0'
+  VERSION = '0.6.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: twitterscraper-ruby
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - ts-3156