twitterscraper-ruby 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3b968497864febb3a245e9d3ce42cc7fe948bfc3c1995401597cfcadca352b46
-  data.tar.gz: fd23f719a5dd760a397e936bd9cbea3a8c4d9de9b380728f954da11b753f0531
+  metadata.gz: 6791ebfd82694e768350ec33a19d3a34336c26c5344e57ef92af6bf02a0dddf1
+  data.tar.gz: 351bf02ad483c60993114a828a4a1e39b936ab8e1373237aa66de6d0a3c809a6
 SHA512:
-  metadata.gz: a080abd711c46c34d366525acbf1f7f41db5b6c81110be0d615bf9a73401129328f24536278da96f54e806be910e4ac7e6f9dbd74a4a0bebac6352d5c808a7e2
-  data.tar.gz: d49db2835fb8ccb4b491bea050c2fd0024569b3788c873c126ce62b27951362223c381b81d9d8890281bb50fff26b2c923c3040dee6b1a0f35394c0bc236d3a2
+  metadata.gz: 236d01eaaf4ed8c5c016fff35b5794e1609e840d8d27edfba92e0fc63138dfced1a10f4e2952d919360d9cc5111bc4987422a9e709c3e798434614b734b3b029
+  data.tar.gz: 6831c32b358651e8c75af0772afbd0f2888934e5ef314112ecaa2dab1bcaeb681dc6a350d473eab79e36a83da57059b35dd88d693cb3a2f894789cb03ceb1e8c

data/.ruby-version

@@ -0,0 +1 @@
+2.6.4

data/Gemfile.lock

@@ -1,12 +1,16 @@
 PATH
   remote: .
   specs:
-    twitterscraper-ruby (0.1.0)
+    twitterscraper-ruby (0.2.0)
+      nokogiri
 
 GEM
   remote: https://rubygems.org/
   specs:
+    mini_portile2 (2.4.0)
     minitest (5.14.1)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
     rake (12.3.3)
 
 PLATFORMS

data/README.md

@@ -1,4 +1,4 @@
-# Twitterscraper::Ruby
+# twitterscraper-ruby
 
 Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/twitterscraper/ruby`. To experiment with that code, run `bin/console` for an interactive prompt.
 
@@ -22,7 +22,9 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+```ruby
+require 'twitterscraper'
+```
 
 ## Development
 

data/lib/twitterscraper.rb

@@ -1,6 +1,26 @@
-require "version"
+require 'twitterscraper/logger'
+require 'twitterscraper/proxy'
+require 'twitterscraper/http'
+require 'twitterscraper/lang'
+require 'twitterscraper/query'
+require 'twitterscraper/client'
+require 'twitterscraper/tweet'
+require 'version'
 
 module Twitterscraper
   class Error < StandardError; end
   # Your code goes here...
+
+  def self.logger
+    @logger ||= ::Logger.new(STDOUT)
+  end
+
+  def self.logger=(logger)
+    if logger.nil?
+      self.logger.level = ::Logger::FATAL
+      return self.logger
+    end
+
+    @logger = logger
+  end
 end

data/lib/twitterscraper/client.rb

@@ -0,0 +1,5 @@
+module Twitterscraper
+  class Client
+    include Query
+  end
+end

data/lib/twitterscraper/http.rb

@@ -0,0 +1,30 @@
+module Twitterscraper
+  module Http
+
+    module_function
+
+    def get(url, headers = {}, proxy = nil, timeout = nil)
+      timeout ||= 3
+
+      if proxy
+        ip, port = proxy.split(':')
+        http_class = Net::HTTP::Proxy(ip, port.to_i)
+      else
+        http_class = Net::HTTP
+      end
+
+      uri = URI.parse(url)
+      http = http_class.new(uri.host, uri.port)
+      http.use_ssl = true if url.match?(/^https/)
+      http.open_timeout = timeout
+      http.read_timeout = timeout
+      req = Net::HTTP::Get.new(uri)
+
+      headers.each do |key, value|
+        req[key] = value
+      end
+
+      http.request(req).body
+    end
+  end
+end

data/lib/twitterscraper/lang.rb

@@ -0,0 +1,40 @@
+module Twitterscraper
+  class Lang
+    LIST = [
+        'en', # English
+        'ar', # Arabic
+        'bn', # Bengali
+        'cs', # Czech
+        'da', # Danish
+        'de', # German
+        'el', # Greek
+        'es', # Spanish
+        'fa', # Persian
+        'fi', # Finnish
+        'fil', # Filipino
+        'fr', # French
+        'he', # Hebrew
+        'hi', # Hindi
+        'hu', # Hungarian
+        'id', # Indonesian
+        'it', # Italian
+        'ja', # Japanese
+        'ko', # Korean
+        'msa', # Malay
+        'nl', # Dutch
+        'no', # Norwegian
+        'pl', # Polish
+        'pt', # Portuguese
+        'ro', # Romanian
+        'ru', # Russian
+        'sv', # Swedish
+        'th', # Thai
+        'tr', # Turkish
+        'uk', # Ukranian
+        'ur', # Urdu
+        'vi', # Vietnamese
+        'zh-cn', # Chinese Simplified
+        'zh-tw', # Chinese Traditional
+    ]
+  end
+end

data/lib/twitterscraper/logger.rb

@@ -0,0 +1,9 @@
+require 'logger'
+
+module Twitterscraper
+  module Logger
+    def logger
+      Twitterscraper.logger
+    end
+  end
+end

data/lib/twitterscraper/proxy.rb

@@ -0,0 +1,33 @@
+module Twitterscraper
+  module Proxy
+
+    PROXY_URL = 'https://free-proxy-list.net/'
+
+    class RetryExhausted < StandardError
+    end
+
+    module_function
+
+    def get_proxies(retries = 3)
+      response = Twitterscraper::Http.get(PROXY_URL)
+      html = Nokogiri::HTML(response)
+      table = html.xpath('//*[@id="proxylisttable"]').first
+
+      proxies = []
+
+      table.xpath('tbody/tr').each do |tr|
+        cells = tr.xpath('td')
+        ip, port = cells[0].text.strip, cells[1].text.strip
+        proxies << ip + ':' + port
+      end
+
+      proxies
+    rescue => e
+      if (retries -= 1) > 0
+        retry
+      else
+        raise RetryExhausted.new(e.inspect)
+      end
+    end
+  end
+end

data/lib/twitterscraper/query.rb

@@ -0,0 +1,147 @@
+require 'net/http'
+require 'nokogiri'
+require 'date'
+require 'json'
+
+module Twitterscraper
+  module Query
+    include Logger
+
+    USER_AGENT_LIST = [
+        'Mozilla/5.0 (Windows; U; Windows NT 6.1; x64; fr; rv:1.9.2.13) Gecko/20101203 Firebird/3.6.13',
+        'Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko',
+        'Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:2.2) Gecko/20110201',
+        'Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16',
+        'Mozilla/5.0 (Windows NT 5.2; RW; rv:7.0a1) Gecko/20091211 SeaMonkey/9.23a1pre',
+    ]
+    USER_AGENT = USER_AGENT_LIST.sample
+
+    INIT_URL = 'https://twitter.com/search?f=tweets&vertical=default&q=__QUERY__&l=__LANG__'
+    RELOAD_URL = 'https://twitter.com/i/search/timeline?f=tweets&vertical=' +
+        'default&include_available_features=1&include_entities=1&' +
+        'reset_error_state=false&src=typd&max_position=__POS__&q=__QUERY__&l=__LANG__'
+    INIT_URL_USER = 'https://twitter.com/{u}'
+    RELOAD_URL_USER = 'https://twitter.com/i/profiles/show/{u}/timeline/tweets?' +
+        'include_available_features=1&include_entities=1&' +
+        'max_position={pos}&reset_error_state=false'
+
+    def get_query_url(query, lang, pos, from_user = false)
+      # if from_user
+      #   if !pos
+      #     INIT_URL_USER.format(u = query)
+      #   else
+      #     RELOAD_URL_USER.format(u = query, pos = pos)
+      #   end
+      # end
+      if pos
+        RELOAD_URL.sub('__QUERY__', query).sub('__LANG__', lang.to_s).sub('__POS__', pos)
+      else
+        INIT_URL.sub('__QUERY__', query).sub('__LANG__', lang.to_s)
+      end
+    end
+
+    def query_single_page(query, lang, pos, retries = 30, from_user = false, timeout = 3, headers: [], proxies: [])
+      query = query.gsub(' ', '%20').gsub('#', '%23').gsub(':', '%3A').gsub('&', '%26')
+      logger.info("Querying #{query}")
+
+      url = get_query_url(query, lang, pos, from_user)
+      logger.debug("Scraping tweets from #{url}")
+
+      response = nil
+      begin
+        proxy = proxies.sample
+        logger.info("Using proxy #{proxy}")
+
+        response = Twitterscraper::Http.get(url, headers, proxy, timeout)
+      rescue => e
+        logger.debug "query_single_page: #{e.inspect}"
+        if (retries -= 1) > 0
+          logger.info("Retrying... (Attempts left: #{retries - 1})")
+          retry
+        else
+          raise
+        end
+      end
+
+      html = ''
+      json_resp = nil
+
+      if pos
+        begin
+          json_resp = JSON.parse(response)
+          html = json_resp['items_html'] || ''
+        rescue => e
+          logger.warn("Failed to parse JSON #{e.inspect} while requesting #{url}")
+        end
+      else
+        html = response || ''
+      end
+
+      tweets = Tweet.from_html(html)
+
+      if tweets.empty?
+        if json_resp && json_resp['has_more_items']
+          pos = json_resp['min_position']
+        else
+          pos = nil
+        end
+        return [], pos
+      end
+
+      if json_resp
+        [tweets, json_resp['min_position']]
+      elsif from_user
+        raise NotImplementedError
+      else
+        [tweets, "TWEET-#{tweets[-1].tweet_id}-#{tweets[0].tweet_id}"]
+      end
+    end
+
+    def query_tweets(query, start_date: nil, end_date: nil, limit: 100, threads: 2, lang: '')
+      start_date = start_date ? Date.parse(start_date) : Date.parse('2006-3-21')
+      end_date = end_date ? Date.parse(end_date) : Date.today
+      if start_date == end_date
+        raise 'Please specify different values for :start_date and :end_date.'
+      elsif start_date > end_date
+        raise 'The :start_date must occur before :end_date.'
+      end
+
+      # TODO parallel
+
+      pos = nil
+      all_tweets = []
+
+      proxies = Twitterscraper::Proxy.get_proxies
+      logger.info "Using #{proxies.size} proxies"
+
+      headers = {'User-Agent': USER_AGENT, 'X-Requested-With': 'XMLHttpRequest'}
+      logger.info("Headers #{headers}")
+
+      start_date.upto(end_date) do |date|
+        break if date == end_date
+
+        queries = query + " since:#{date} until:#{date + 1}"
+
+        while true
+          new_tweets, new_pos = query_single_page(queries, lang, pos, headers: headers, proxies: proxies)
+          logger.info("Got #{new_tweets.size} tweets")
+          logger.debug("new_pos=#{new_pos}")
+
+          unless new_tweets.empty?
+            all_tweets.concat(new_tweets)
+            all_tweets.uniq! { |t| t.tweet_id }
+          end
+
+          break unless new_pos
+          break if all_tweets.size >= limit
+
+          pos = new_pos
+        end
+
+        break if all_tweets.size >= limit
+      end
+
+      all_tweets
+    end
+  end
+end

data/lib/twitterscraper/tweet.rb

@@ -0,0 +1,41 @@
+require 'time'
+
+module Twitterscraper
+  class Tweet
+    attr_reader :screen_name, :name, :user_id, :tweet_id, :tweet_url, :timestamp, :created_at, :text
+
+    def initialize(attrs)
+      attrs.each do |key, value|
+        instance_variable_set("@#{key}", value)
+      end
+    end
+
+    class << self
+      def from_html(text)
+        html = Nokogiri::HTML(text)
+        from_tweets_html(html.xpath("//li[@class[contains(., 'js-stream-item')]]/div[@class[contains(., 'js-stream-tweet')]]"))
+      end
+
+      def from_tweets_html(html)
+        html.map do |tweet|
+          from_tweet_html(tweet)
+        end
+      end
+
+      def from_tweet_html(html)
+        inner_html = Nokogiri::HTML(html.inner_html)
+        timestamp = inner_html.xpath("//span[@class[contains(., 'js-short-timestamp')]]").first.attr('data-time').to_i
+        new(
+            screen_name: html.attr('data-screen-name'),
+            name: html.attr('data-name'),
+            user_id: html.attr('data-user-id').to_i,
+            tweet_id: html.attr('data-tweet-id').to_i,
+            tweet_url: 'https://twitter.com' + html.attr('data-permalink-path'),
+            timestamp: timestamp,
+            created_at: Time.at(timestamp, in: '+00:00'),
+            text: inner_html.xpath("//div[@class[contains(., 'js-tweet-text-container')]]/p[@class[contains(., 'js-tweet-text')]]").first.text,
+        )
+      end
+    end
+  end
+end

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Twitterscraper
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/twitterscraper-ruby.gemspec

@@ -24,4 +24,6 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
+
+  spec.add_dependency "nokogiri"
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: twitterscraper-ruby
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - ts-3156
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-11 00:00:00.000000000 Z
-dependencies: []
+date: 2020-07-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A gem  to scrape Tweets
 email:
 - ts_3156@yahoo.co.jp
@@ -19,6 +33,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".irbrc"
+- ".ruby-version"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -30,6 +45,13 @@ files:
 - bin/setup
 - lib/twitterscraper-ruby.rb
 - lib/twitterscraper.rb
+- lib/twitterscraper/client.rb
+- lib/twitterscraper/http.rb
+- lib/twitterscraper/lang.rb
+- lib/twitterscraper/logger.rb
+- lib/twitterscraper/proxy.rb
+- lib/twitterscraper/query.rb
+- lib/twitterscraper/tweet.rb
 - lib/version.rb
 - twitterscraper-ruby.gemspec
 homepage: https://github.com/ts-3156/twitterscraper-ruby
@@ -54,8 +76,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: A gem  to scrape Tweets