twitterscraper-ruby 0.15.0

data/lib/twitterscraper-ruby.rb

@@ -0,0 +1 @@
+require_relative "./twitterscraper"

data/lib/twitterscraper.rb

@@ -0,0 +1,27 @@
+require 'twitterscraper/logger'
+require 'twitterscraper/proxy'
+require 'twitterscraper/http'
+require 'twitterscraper/lang'
+require 'twitterscraper/cache'
+require 'twitterscraper/query'
+require 'twitterscraper/client'
+require 'twitterscraper/tweet'
+require 'twitterscraper/template'
+require 'version'
+
+module Twitterscraper
+  class Error < StandardError; end
+
+  def self.logger
+    @logger ||= ::Logger.new(STDOUT, level: ::Logger::INFO)
+  end
+
+  def self.logger=(logger)
+    if logger.nil?
+      self.logger.level = ::Logger::FATAL
+      return self.logger
+    end
+
+    @logger = logger
+  end
+end

data/lib/twitterscraper/cache.rb

@@ -0,0 +1,69 @@
+require 'base64'
+require 'digest/md5'
+
+module Twitterscraper
+  class Cache
+    def initialize()
+      @ttl = 3600 # 1 hour
+      @dir = 'cache'
+      Dir.mkdir(@dir) unless File.exist?(@dir)
+    end
+
+    def read(key)
+      key = cache_key(key)
+      file = File.join(@dir, key)
+      entry = Entry.from_json(File.read(file))
+      entry.value if entry.time > Time.now - @ttl
+    rescue Errno::ENOENT => e
+      nil
+    end
+
+    def write(key, value)
+      key = cache_key(key)
+      entry = Entry.new(key, value, Time.now)
+      file = File.join(@dir, key)
+      File.write(file, entry.to_json)
+    end
+
+    def fetch(key, &block)
+      if (value = read(key))
+        value
+      else
+        yield.tap { |v| write(key, v) }
+      end
+    end
+
+    def cache_key(key)
+      value = key.gsub(':', '%3A').gsub('/', '%2F').gsub('?', '%3F').gsub('=', '%3D').gsub('&', '%26')
+      value = Digest::MD5.hexdigest(value) if value.length >= 100
+      value
+    end
+
+    class Entry < Hash
+      attr_reader :key, :value, :time
+
+      def initialize(key, value, time)
+        @key = key
+        @value = value
+        @time = time
+      end
+
+      def attrs
+        {key: @key, value: @value, time: @time}
+      end
+
+      def to_json
+        hash = attrs
+        hash[:value] = Base64.encode64(hash[:value])
+        hash.to_json
+      end
+
+      class << self
+        def from_json(text)
+          json = JSON.parse(text)
+          new(json['key'], Base64.decode64(json['value']), Time.parse(json['time']))
+        end
+      end
+    end
+  end
+end

data/lib/twitterscraper/cli.rb

@@ -0,0 +1,119 @@
+$stdout.sync = true
+
+require 'json'
+require 'optparse'
+require 'twitterscraper'
+
+module Twitterscraper
+  class Cli
+    def parse
+      @options = parse_options(ARGV)
+      initialize_logger
+    end
+
+    def run
+      print_help || return if print_help?
+      print_version || return if print_version?
+
+      query_options = {
+          type: options['type'],
+          start_date: options['start_date'],
+          end_date: options['end_date'],
+          lang: options['lang'],
+          limit: options['limit'],
+          daily_limit: options['daily_limit'],
+          order: options['order'],
+          threads: options['threads'],
+      }
+      client = Twitterscraper::Client.new(cache: options['cache'], proxy: options['proxy'])
+      tweets = client.query_tweets(options['query'], query_options)
+      export(tweets) unless tweets.empty?
+    end
+
+    def export(tweets)
+      write_json = lambda { File.write(options['output'], generate_json(tweets)) }
+
+      if options['format'] == 'json'
+        write_json.call
+      elsif options['format'] == 'html'
+        File.write('tweets.html', Template.tweets_embedded_html(tweets))
+      else
+        write_json.call
+      end
+    end
+
+    def generate_json(tweets)
+      if options['pretty']
+        ::JSON.pretty_generate(tweets)
+      else
+        ::JSON.generate(tweets)
+      end
+    end
+
+    def options
+      @options
+    end
+
+    def parse_options(argv)
+      options = argv.getopts(
+          'h',
+          'help',
+          'v',
+          'version',
+          'type:',
+          'query:',
+          'start_date:',
+          'end_date:',
+          'lang:',
+          'limit:',
+          'daily_limit:',
+          'order:',
+          'threads:',
+          'output:',
+          'format:',
+          'cache:',
+          'proxy:',
+          'pretty',
+          'verbose',
+      )
+
+      options['type'] ||= 'search'
+      options['start_date'] = Query::OLDEST_DATE if options['start_date'] == 'oldest'
+      options['lang'] ||= ''
+      options['limit'] = (options['limit'] || 100).to_i
+      options['daily_limit'] = options['daily_limit'].to_i if options['daily_limit']
+      options['threads'] = (options['threads'] || 2).to_i
+      options['format'] ||= 'json'
+      options['order'] ||= 'desc'
+      options['output'] ||= "tweets.#{options['format']}"
+
+      options['cache'] = options['cache'] != 'false'
+      options['proxy'] = options['proxy'] != 'false'
+
+      options
+    end
+
+    def initialize_logger
+      Twitterscraper.logger.level = ::Logger::DEBUG if options['verbose']
+    end
+
+    def print_help?
+      options['h'] || options['help']
+    end
+
+    def print_help
+      puts <<~'SHELL'
+        Usage:
+          twitterscraper --query KEYWORD --limit 100 --threads 10 --start_date 2020-07-01 --end_date 2020-07-10 --lang ja --proxy --output output.json
+      SHELL
+    end
+
+    def print_version?
+      options['v'] || options['version']
+    end
+
+    def print_version
+      puts "twitterscraper-#{VERSION}"
+    end
+  end
+end

data/lib/twitterscraper/client.rb

@@ -0,0 +1,18 @@
+module Twitterscraper
+  class Client
+    include Query
+
+    def initialize(cache: true, proxy: true)
+      @cache = cache
+      @proxy = proxy
+    end
+
+    def cache_enabled?
+      @cache
+    end
+
+    def proxy_enabled?
+      @proxy
+    end
+  end
+end

data/lib/twitterscraper/http.rb

@@ -0,0 +1,31 @@
+module Twitterscraper
+  module Http
+
+    module_function
+
+    def get(url, headers = {}, proxy = nil, timeout = nil)
+      timeout ||= 3
+
+      if proxy
+        ip, port = proxy.split(':')
+        http_class = Net::HTTP::Proxy(ip, port.to_i)
+      else
+        http_class = Net::HTTP
+      end
+
+      uri = URI.parse(url)
+      http = http_class.new(uri.host, uri.port)
+      http.use_ssl = true if url.match?(/^https/)
+      http.open_timeout = timeout
+      http.read_timeout = timeout
+      req = Net::HTTP::Get.new(uri)
+
+      headers.each do |key, value|
+        req[key] = value
+      end
+
+      res = http.start { http.request(req) }
+      res.body
+    end
+  end
+end

data/lib/twitterscraper/lang.rb

@@ -0,0 +1,40 @@
+module Twitterscraper
+  class Lang
+    LIST = [
+        'en', # English
+        'ar', # Arabic
+        'bn', # Bengali
+        'cs', # Czech
+        'da', # Danish
+        'de', # German
+        'el', # Greek
+        'es', # Spanish
+        'fa', # Persian
+        'fi', # Finnish
+        'fil', # Filipino
+        'fr', # French
+        'he', # Hebrew
+        'hi', # Hindi
+        'hu', # Hungarian
+        'id', # Indonesian
+        'it', # Italian
+        'ja', # Japanese
+        'ko', # Korean
+        'msa', # Malay
+        'nl', # Dutch
+        'no', # Norwegian
+        'pl', # Polish
+        'pt', # Portuguese
+        'ro', # Romanian
+        'ru', # Russian
+        'sv', # Swedish
+        'th', # Thai
+        'tr', # Turkish
+        'uk', # Ukranian
+        'ur', # Urdu
+        'vi', # Vietnamese
+        'zh-cn', # Chinese Simplified
+        'zh-tw', # Chinese Traditional
+    ]
+  end
+end

data/lib/twitterscraper/logger.rb

@@ -0,0 +1,9 @@
+require 'logger'
+
+module Twitterscraper
+  module Logger
+    def logger
+      Twitterscraper.logger
+    end
+  end
+end

data/lib/twitterscraper/proxy.rb

@@ -0,0 +1,65 @@
+module Twitterscraper
+  module Proxy
+
+    PROXY_URL = 'https://free-proxy-list.net/'
+
+    class RetryExhausted < StandardError
+    end
+
+    class Pool
+      def initialize
+        @items = Proxy.get_proxies
+        @cur_index = 0
+      end
+
+      def sample
+        if @cur_index >= @items.size
+          reload
+        end
+        @cur_index += 1
+        @items[@cur_index - 1]
+      end
+
+      def size
+        @items.size
+      end
+
+      def empty?
+        @items.empty?
+      end
+
+      private
+
+      def reload
+        @items = Proxy.get_proxies
+        @cur_index = 0
+      end
+    end
+
+    module_function
+
+    def get_proxies(retries = 3)
+      response = Twitterscraper::Http.get(PROXY_URL)
+      html = Nokogiri::HTML(response)
+      table = html.xpath('//table[@id="proxylisttable"]').first
+
+      proxies = []
+
+      table.xpath('tbody/tr').each do |tr|
+        cells = tr.xpath('td')
+        ip, port, anonymity, https = [0, 1, 4, 6].map { |i| cells[i].text.strip }
+        next unless ['elite proxy', 'anonymous'].include?(anonymity)
+        next if https == 'no'
+        proxies << ip + ':' + port
+      end
+
+      proxies.shuffle
+    rescue => e
+      if (retries -= 1) > 0
+        retry
+      else
+        raise RetryExhausted.new(e.inspect)
+      end
+    end
+  end
+end

data/lib/twitterscraper/query.rb

@@ -0,0 +1,254 @@
+require 'resolv-replace'
+require 'net/http'
+require 'nokogiri'
+require 'date'
+require 'json'
+require 'erb'
+require 'parallel'
+
+module Twitterscraper
+  module Query
+    include Logger
+
+    USER_AGENT_LIST = [
+        'Mozilla/5.0 (Windows; U; Windows NT 6.1; x64; fr; rv:1.9.2.13) Gecko/20101203 Firebird/3.6.13',
+        'Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko',
+        'Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:2.2) Gecko/20110201',
+        'Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16',
+        'Mozilla/5.0 (Windows NT 5.2; RW; rv:7.0a1) Gecko/20091211 SeaMonkey/9.23a1pre',
+    ]
+
+    INIT_URL = 'https://twitter.com/search?f=tweets&vertical=default&q=__QUERY__&l=__LANG__'
+    RELOAD_URL = 'https://twitter.com/i/search/timeline?f=tweets&vertical=' +
+        'default&include_available_features=1&include_entities=1&' +
+        'reset_error_state=false&src=typd&max_position=__POS__&q=__QUERY__&l=__LANG__'
+    INIT_URL_USER = 'https://twitter.com/__USER__'
+    RELOAD_URL_USER = 'https://twitter.com/i/profiles/show/__USER__/timeline/tweets?' +
+        'include_available_features=1&include_entities=1&' +
+        'max_position=__POS__&reset_error_state=false'
+
+    def build_query_url(query, lang, from_user, pos)
+      if from_user
+        if pos
+          RELOAD_URL_USER.sub('__USER__', query).sub('__POS__', pos.to_s)
+        else
+          INIT_URL_USER.sub('__USER__', query)
+        end
+      else
+        if pos
+          RELOAD_URL.sub('__QUERY__', query).sub('__LANG__', lang.to_s).sub('__POS__', pos)
+        else
+          INIT_URL.sub('__QUERY__', query).sub('__LANG__', lang.to_s)
+        end
+      end
+    end
+
+    def get_single_page(url, headers, proxies, timeout = 6, retries = 30)
+      return nil if stop_requested?
+      unless proxies.empty?
+        proxy = proxies.sample
+        logger.info("Using proxy #{proxy}")
+      end
+      Http.get(url, headers, proxy, timeout)
+    rescue => e
+      logger.debug "query_single_page: #{e.inspect}"
+      if (retries -= 1) > 0
+        logger.info "Retrying... (Attempts left: #{retries - 1})"
+        retry
+      else
+        raise Error.new("#{e.inspect} url=#{url}")
+      end
+    end
+
+    def parse_single_page(text, html = true)
+      return [nil, nil] if text.nil? || text == ''
+
+      if html
+        json_resp = nil
+        items_html = text
+      else
+        json_resp = JSON.parse(text)
+        items_html = json_resp['items_html'] || ''
+        logger.warn json_resp['message'] if json_resp['message'] # Sorry, you are rate limited.
+      end
+
+      [items_html, json_resp]
+    end
+
+    def query_single_page(query, lang, type, pos, headers: [], proxies: [])
+      logger.info "Querying #{query}"
+      query = ERB::Util.url_encode(query)
+
+      url = build_query_url(query, lang, type == 'user', pos)
+      http_request = lambda do
+        logger.debug "Scraping tweets from #{url}"
+        get_single_page(url, headers, proxies)
+      end
+
+      if cache_enabled?
+        client = Cache.new
+        if (response = client.read(url))
+          logger.debug 'Fetching tweets from cache'
+        else
+          response = http_request.call
+          client.write(url, response) unless stop_requested?
+        end
+      else
+        response = http_request.call
+      end
+      return [], nil if response.nil? || response.empty?
+
+      html, json_resp = parse_single_page(response, pos.nil?)
+
+      tweets = Tweet.from_html(html)
+
+      if tweets.empty?
+        return [], (json_resp && json_resp['has_more_items'] && json_resp['min_position'])
+      end
+
+      if json_resp
+        [tweets, json_resp['min_position']]
+      elsif type
+        [tweets, tweets[-1].tweet_id]
+      else
+        [tweets, "TWEET-#{tweets[-1].tweet_id}-#{tweets[0].tweet_id}"]
+      end
+    end
+
+    OLDEST_DATE = Date.parse('2006-03-21')
+
+    def validate_options!(queries, type:, start_date:, end_date:, lang:, limit:, threads:)
+      query = queries[0]
+      if query.nil? || query == ''
+        raise Error.new('Please specify a search query.')
+      end
+
+      if ERB::Util.url_encode(query).length >= 500
+        raise Error.new(':query must be a UTF-8, URL-encoded search query of 500 characters maximum, including operators.')
+      end
+
+      if start_date && end_date
+        if start_date == end_date
+          raise Error.new('Please specify different values for :start_date and :end_date.')
+        elsif start_date > end_date
+          raise Error.new(':start_date must occur before :end_date.')
+        end
+      end
+
+      if start_date
+        if start_date < OLDEST_DATE
+          raise Error.new(":start_date must be greater than or equal to #{OLDEST_DATE}")
+        end
+      end
+
+      if end_date
+        today = Date.today
+        if end_date > Date.today
+          raise Error.new(":end_date must be less than or equal to today(#{today})")
+        end
+      end
+    end
+
+    def build_queries(query, start_date, end_date)
+      if start_date && end_date
+        date_range = start_date.upto(end_date - 1)
+        date_range.map { |date| query + " since:#{date} until:#{date + 1}" }
+      elsif start_date
+        [query + " since:#{start_date}"]
+      elsif end_date
+        [query + " until:#{end_date}"]
+      else
+        [query]
+      end
+    end
+
+    def main_loop(query, lang, type, limit, daily_limit, headers, proxies)
+      pos = nil
+      daily_tweets = []
+
+      while true
+        new_tweets, new_pos = query_single_page(query, lang, type, pos, headers: headers, proxies: proxies)
+        unless new_tweets.empty?
+          daily_tweets.concat(new_tweets)
+          daily_tweets.uniq! { |t| t.tweet_id }
+
+          @mutex.synchronize {
+            @all_tweets.concat(new_tweets)
+            @all_tweets.uniq! { |t| t.tweet_id }
+          }
+        end
+        logger.info "Got #{new_tweets.size} tweets (total #{@all_tweets.size})"
+
+        break unless new_pos
+        break if daily_limit && daily_tweets.size >= daily_limit
+        break if @all_tweets.size >= limit
+
+        pos = new_pos
+      end
+
+      if !@stop_requested && @all_tweets.size >= limit
+        logger.warn "The limit you specified has been reached limit=#{limit} tweets=#{@all_tweets.size}"
+        @stop_requested = true
+      end
+    end
+
+    def stop_requested?
+      @stop_requested
+    end
+
+    def query_tweets(query, type: 'search', start_date: nil, end_date: nil, lang: nil, limit: 100, daily_limit: nil, order: 'desc', threads: 2)
+      start_date = Date.parse(start_date) if start_date && start_date.is_a?(String)
+      end_date = Date.parse(end_date) if end_date && end_date.is_a?(String)
+      queries = build_queries(query, start_date, end_date)
+      if threads > queries.size
+        logger.warn 'The maximum number of :threads is the number of dates between :start_date and :end_date.'
+        threads = queries.size
+      end
+      if proxy_enabled?
+        proxies = Proxy::Pool.new
+        logger.debug "Fetch #{proxies.size} proxies"
+      else
+        proxies = []
+        logger.debug 'Proxy disabled'
+      end
+      logger.debug "Cache #{cache_enabled? ? 'enabled' : 'disabled'}"
+
+
+      validate_options!(queries, type: type, start_date: start_date, end_date: end_date, lang: lang, limit: limit, threads: threads)
+
+      logger.info "The number of threads #{threads}"
+
+      headers = {'User-Agent': USER_AGENT_LIST.sample, 'X-Requested-With': 'XMLHttpRequest'}
+      logger.info "Headers #{headers}"
+
+      @all_tweets = []
+      @mutex = Mutex.new
+      @stop_requested = false
+
+      if threads > 1
+        Thread.abort_on_exception = true
+        logger.debug "Set 'Thread.abort_on_exception' to true"
+
+        Parallel.each(queries, in_threads: threads) do |query|
+          main_loop(query, lang, type, limit, daily_limit, headers, proxies)
+          raise Parallel::Break if stop_requested?
+        end
+      else
+        queries.each do |query|
+          main_loop(query, lang, type, limit, daily_limit, headers, proxies)
+          break if stop_requested?
+        end
+      end
+
+      @all_tweets.sort_by { |tweet| (order == 'desc' ? -1 : 1) * tweet.created_at.to_i }
+    end
+
+    def search(query, start_date: nil, end_date: nil, lang: '', limit: 100, daily_limit: nil, order: 'desc', threads: 2)
+      query_tweets(query, type: 'search', start_date: start_date, end_date: end_date, lang: lang, limit: limit, daily_limit: daily_limit, order: order, threads: threads)
+    end
+
+    def user_timeline(screen_name, limit: 100, order: 'desc')
+      query_tweets(screen_name, type: 'user', start_date: nil, end_date: nil, lang: nil, limit: limit, daily_limit: nil, order: order, threads: 1)
+    end
+  end
+end