goldtweets 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d05e029b4a06fa88926306487069f5778f83a3eb40f6ef714004636319d0ed26
+  data.tar.gz: f645ec888e05149aa30c387f8bd763680ac6869edd643d112d32ef4fec60f91e
+SHA512:
+  metadata.gz: a7326365bed9d11762dd2ecc63f32e600dd60545b709decf60a605b0a0716528121ca61ad9ab76841da4d313ad6bf20e18d36a42e3fd19f30aae83a8f97d0418
+  data.tar.gz: f18dfbc3e5bf8155c2213fcf36aa2a3f9fc99908ed51fab79535f32e3bf5c77d9b10e4ac5c8abdd2c98a48978df4e88c072ac267f2a6cb486ade7a6d6e6ea6e8

data/lib/goldtweets.rb

@@ -0,0 +1,10 @@
+require 'goldtweets/client'
+require 'goldtweets/search'
+require 'goldtweets/tweet'
+
+module GoldTweets
+  # Convenience method, identical to calling GoldTweets::Client.get_tweets
+  def self.get_tweets(criteria)
+    ::GoldTweets::Client.get_tweets(criteria)
+  end
+end

data/lib/goldtweets/client.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'nokogiri'
+require 'net/http'
+require 'uri'
+
+require 'goldtweets/tweet'
+
+module GoldTweets
+  module Client
+    # User agents to present to Twitter search
+    USER_AGENTS = [ 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:63.0) Gecko/20100101 Firefox/63.0',
+                    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:62.0) Gecko/20100101 Firefox/62.0',
+                    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:61.0) Gecko/20100101 Firefox/61.0',
+                    'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:63.0) Gecko/20100101 Firefox/63.0',
+                    'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
+                    'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
+                    'Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko',
+                    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Safari/605.1.15'
+                  ].freeze
+
+    # Static list of headers to be sent with API requests
+    DEFAULT_HEADERS = { 'Host' => 'twitter.com',
+                        'Accept' => 'application/json, text/javascript, */*; q=0.01',
+                        'Accept-Language' => 'en-US,en;q=0.5',
+                        'X-Requested-With' => 'XMLHttpRequest',
+                        'Connection' => 'keep-alive'
+                      }.freeze
+    # How many usernames to put in a single search
+    USERNAMES_PER_BATCH = 20
+
+    # URLs for searching and generating permalinks back to tweets
+    SEARCH_PREFIX = 'https://twitter.com/i/search/timeline?'
+    PERMALINK_PREFIX = 'https://twitter.com'
+
+    # Static list of parameters sent with a search
+    DEFAULT_PARAMETERS = { 'vertical' => 'news',
+                           'src' => 'typd',
+                           'include_available_features' => '1',
+                           'include_entities' => '1',
+                           'reset_error_state' => 'false'
+                         }.freeze
+
+    # XPath selectors
+    TWEETS_SELECTOR    = "//div[contains(concat(' ', normalize-space(@class), ' '), ' js-stream-tweet ') and not(contains(concat(' ', normalize-space(@class), ' '), ' withheld-tweet '))]"
+    USERNAMES_SELECTOR = ".//span[contains(concat(' ', normalize-space(@class), ' '), ' username ') and contains(concat(' ', normalize-space(@class), ' '), ' u-dir ')]/b"
+    AUTHORID_SELECTOR  = ".//a[contains(concat(' ', normalize-space(@class), ' '), ' js-user-profile-link ')]"
+    CONTENT_SELECTOR   = ".//p[contains(concat(' ', normalize-space(@class), ' '), ' js-tweet-text ')]"
+    RETWEETS_SELECTOR  = ".//span[contains(concat(' ', normalize-space(@class), ' '), ' ProfileTweet-action--retweet ')]/span[contains(concat(' ', normalize-space(@class), ' '), ' ProfileTweet-actionCount ')]"
+    FAVORITES_SELECTOR = ".//span[contains(concat(' ', normalize-space(@class), ' '), ' ProfileTweet-action--favorite ')]/span[contains(concat(' ', normalize-space(@class), ' '), ' ProfileTweet-actionCount ')]"
+    REPLIES_SELECTOR   = ".//span[contains(concat(' ', normalize-space(@class), ' '), ' ProfileTweet-action--reply ')]/span[contains(concat(' ', normalize-space(@class), ' '), ' ProfileTweet-actionCount ')]"
+    TIMESTAMP_SELECTOR = ".//small[contains(concat(' ', normalize-space(@class), ' '), ' time ')]//span[contains(concat(' ', normalize-space(@class), ' '), ' js-short-timestamp ')]"
+    GEO_SELECTOR       = ".//span[contains(concat(' ', normalize-space(@class), ' '), ' Tweet-geo ')]"
+    LINK_SELECTOR      = ".//a"
+
+    # Interim response structure useful for tweet fetch and processing logic
+    Response = Struct.new(:body, :new_cursor, :new_cookies, :more_items)
+
+    # Fetch tweets based on a GoldTweets::Search object
+    # This functionality is presently lacking several features of the original
+    # python library - proxy support, emoji handling, and allowing a provided
+    # block to be run on tweets as they are processed among them.
+    def self.get_tweets(criteria)
+      user_agent = USER_AGENTS.sample
+      cookie_jar = ''
+      usernames  = usernames_for(criteria.usernames)
+      batches    = usernames.each_slice(USERNAMES_PER_BATCH).to_a
+
+      batches.map do |batch|
+        refresh_cursor      = ''
+        batch_results_count = 0
+        collected_tweets    = []
+
+        criteria.usernames = batch
+        loop do
+          response       = fetch_tweets(criteria, refresh_cursor, cookie_jar, user_agent)
+          cookie_jar     = response.new_cookies if response.new_cookies
+          refresh_cursor = response.new_cursor
+
+          tweets   = response.body.xpath(TWEETS_SELECTOR).reduce([], &method(:parse_tweet))
+          collected_tweets << tweets
+          batch_results_count += tweets.length
+
+          if (criteria.maximum_tweets.to_i > 0 && batch_results_count >= criteria.maximum_tweets) || (!response.more_items)
+            break
+          end
+        end
+
+        collected_tweets.flatten
+      end.flatten
+    end
+
+    private
+
+    # Coerce usernames into a suitable representation for batching
+    def self.usernames_for(users)
+      case users
+      when Array
+        users.map { |u| u.sub(/^@/, '').downcase }
+      when String
+        [ users.sub(/^@/, '').downcase ]
+      else
+        [[]]
+      end
+    end
+
+    # Function for folding a list of Nokogiri objects fetched from Twitter into
+    # a list of GoldTweets::Tweet objects
+    def self.parse_tweet(tweets, tweet)
+      users    = tweet.xpath(USERNAMES_SELECTOR).map(&:text)
+      return tweets if users.empty?
+
+      message   = tweet.xpath(CONTENT_SELECTOR).map(&method(:sanitize_message)).first
+      rt,f,re   = tweet_interactions(tweet)
+      permalink = PERMALINK_PREFIX + tweet.attr('data-permalink-path')
+      author    = tweet.xpath(AUTHORID_SELECTOR).map { |t| t.attr('data-user-id').to_i }.first
+      timestamp = tweet.xpath(TIMESTAMP_SELECTOR).map { |t| Time.at(t.attr('data-time').to_i) }.first
+      links     = tweet.xpath(LINK_SELECTOR)
+      hts, ats  = tweet_hashtags_and_mentions(links)
+      geo_span  = tweet.xpath(GEO_SELECTOR).map { |t| t.attr('title') }.first.to_s
+      ext_links = links.map { |t| t.attr('data-expanded-url') }.select(&:itself)
+
+      tweet_container           = ::GoldTweets::Tweet.new(users.first)
+      tweet_container.to        = users[1]
+      tweet_container.text      = message
+      tweet_container.retweets  = rt
+      tweet_container.faves     = f
+      tweet_container.replies   = re
+      tweet_container.id        = tweet.attr('data-tweet-id')
+      tweet_container.permalink = permalink
+      tweet_container.author_id = author
+      tweet_container.timestamp = timestamp
+      tweet_container.hashtags  = hts
+      tweet_container.mentions  = ats
+      tweet_container.geo       = geo_span
+      tweet_container.links     = ext_links
+
+      tweets + [tweet_container]
+    end
+
+    # Normalize spacing and remove errant spaces following pound signs, at
+    # signs, and dollar signs
+    def self.sanitize_message(tweet)
+      tweet.text
+           .gsub(/\s+/, ' ')
+           .gsub(/([#@\$]) /, '\1')
+    end
+
+    # Classify interactions (retweets, faves, and replies to a given tweet)
+    def self.tweet_interactions(tweet)
+      [RETWEETS_SELECTOR, FAVORITES_SELECTOR, REPLIES_SELECTOR].map do |selector|
+        tweet.xpath(selector)
+             .map { |node| node.attr('data-tweet-stat-count') }
+             .first
+             .to_i
+      end
+    end
+
+    # Classify links belonging to hashtags and (outgoing) mentions within a
+    # tweet
+    def self.tweet_hashtags_and_mentions(links)
+      links.reduce([[], []]) do |(hashtags, mentions), link|
+        href = link.attr('href')
+        return [hashtags, mentions] unless href.to_s[0] == '/'
+        if link.attr('data-mentioned-user-id')
+          [hashtags, mentions + ['@' + href[1..-1]]]
+        elsif /^\/hashtag\//.match(href)
+          [hashtags + [href.sub(/(?:^\/hashtag\/)/, '#').sub(/(?:\?.*$)/, '')], mentions]
+        else
+          [hashtags, mentions]
+        end
+      end
+    end
+
+    # Perform a search for tweets based on criteria specified
+    def self.fetch_tweets(criteria, refresh_cursor, cookie_jar, user_agent)
+      search   = DEFAULT_PARAMETERS.dup
+      get_data = []
+      search['f'] = 'tweets' unless criteria.top_tweets?
+      search['l'] = criteria.language if criteria.language
+
+      get_data << criteria.query if criteria.query
+      get_data << ([''] + criteria.exclude_words).join(' -')
+      get_data << criteria.username.map { |u| "from:#{u}" }.join(' OR ') if criteria.username
+      get_data << "since:#{criteria.since}" if criteria.since
+      get_data << "until:#{criteria.upto}" if criteria.upto
+      get_data << "min_replies:#{criteria.minimum_replies}" if criteria.minimum_replies
+      get_data << "min_faves:#{criteria.minimum_faves}" if criteria.minimum_faves
+      get_data << "min_retweets:#{criteria.minimum_retweets}" if criteria.minimum_retweets
+
+      if criteria.maximum_distance
+        if criteria.near
+          get_data << "near:#{criteria.near} within:#{criteria.maximum_distance}"
+        elsif criteria.lat && criteria.lon
+          get_data << "geocode:#{criteria.lat},#{criteria.lon},#{criteria.maximum_distance}"
+        end
+      end
+
+      search['q'] = get_data.join(' ').strip
+      search['max_position'] = refresh_cursor
+
+      url = SEARCH_PREFIX + URI.encode_www_form(search)
+      uri = URI(url)
+
+      Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
+        request = Net::HTTP::Get.new(uri)
+        DEFAULT_HEADERS.each { |(k,v)| request[k] = v }
+        request['User-Agent'] = user_agent
+        request['Referer'] = url
+        request['Set-Cookie'] = cookie_jar
+
+        response = http.request(request)
+
+        json        = JSON.parse(response.body)
+        html        = Nokogiri::HTML(json['items_html'])
+        new_cursor  = json['min_position']
+        new_cookies = response['set-cookie']
+        unfinished  = json['has_more_items']
+
+        return Response.new(html, new_cursor, new_cookies, unfinished)
+      end
+    end
+  end
+end

data/lib/goldtweets/search.rb

@@ -0,0 +1,56 @@
+module GoldTweets
+  Search = Struct.new(:username,
+                      :exclude_words,
+                      :since,
+                      :upto,
+                      :minimum_replies,
+                      :minimum_faves,
+                      :minimum_retweets,
+                      :maximum_distance,
+                      :near,
+                      :lat,
+                      :lon,
+                      :query,
+                      :maximum_tweets,
+                      :language,
+                      :emoji,
+                      :top_tweets,
+                      keyword_init: true) do
+
+    # Set default values, otherwise no additional work done here.
+    def initialize(username: nil,
+                   exclude_words: [],
+                   since: nil,
+                   upto: nil,
+                   minimum_replies: nil,
+                   minimum_faves: nil,
+                   minimum_retweets: nil,
+                   maximum_distance: '15mi',
+                   near: nil,
+                   lat: nil,
+                   lon: nil,
+                   query: nil,
+                   maximum_tweets: 0,
+                   language: '',
+                   emoji: :ignore,
+                   top_tweets: false)
+      username = username
+      exclude_words = exclude_words
+      since = since
+      upto = upto
+      minimum_replies = minimum_replies
+      minimum_retweets = minimum_retweets
+      maximum_distance = maximum_distance
+      query = query
+      maximum_tweets = maximum_tweets
+      language = language
+      emoji = emoji
+      top_tweets = top_tweets
+      super
+    end
+
+    alias_method :usernames=, :username=
+    alias_method :usernames, :username
+    alias_method :top_tweets?, :top_tweets
+  end
+end

data/lib/goldtweets/tweet.rb

@@ -0,0 +1,4 @@
+module GoldTweets
+  # Reflects interesting data returned by the API
+  Tweet = Struct.new(:username, :text, :retweets, :faves, :replies, :hashtags, :mentions, :to, :id, :permalink, :author_id, :timestamp, :geo, :links)
+end

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification
+name: goldtweets
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Tina Wuest
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2020-07-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+description: Ruby port of the Python GetOldTweets3 Twitter library
+email: tina@wuest.me
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/goldtweets.rb
+- lib/goldtweets/client.rb
+- lib/goldtweets/search.rb
+- lib/goldtweets/tweet.rb
+homepage: https://gitlab.com/wuest/goldtweets
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key:
+specification_version: 4
+summary: Search Twitter including older tweets
+test_files: []