birdsong 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '03486db7a9efbfce2f7d423f25e33506332bad5a7ceb481da98e68c81b0a75fa'
-  data.tar.gz: 7a46972f68f413a3d3e0cb81b06ece6b9a3a8f9593be5a68664ce0364ab366ec
+  metadata.gz: 9db624502354ba7be08fbe804ff22d1b1bc474fde29ff179126b0781ff42d95a
+  data.tar.gz: bfea616431da77fb69d1fa5587a5e2d7b39d6ca9ef2c09bc81adade2af7de451
 SHA512:
-  metadata.gz: 68eb9d33bac12dc765b776ad32024992640b1363612f99b2cc0b500c6000fd3ef7351d8cc70218d2d21bb681ec40b233b24fdd7062694426891ce3fa22088e62
-  data.tar.gz: 8ceddb1eb88a57dec49116653f0be80be47d29d395894ea35a28aafa84eee72d5aa13deca419e569861f715f5cceff93732473fe5cc9e32ed29ed0cd4ae0eb4b
+  metadata.gz: 2bb4cc4b1ace6b587290aa3952fe97c024939f1ef03e3c903297fb262eb1c4719f99db288c8d7519a14b26a902eb52fc4190fc3cda0a96fac8f8622666da8eab
+  data.tar.gz: d9f5c15d5d2c0577146641785d21041264126b0827a2150a2a94ef9513b72ebbb942f1c62b037e7bb1269143e908dcd8b1bc8c7e40e363496b9d2912cc2dcc0f

data/CHANGELOG.md

@@ -3,3 +3,7 @@
 ## [0.1.0] - 2021-04-27
 
 - Initial release
+
+## [0.2.0] - 2023-10-04
+
+- Fixed to use Selenium for scraping instead of the now defunct API

data/lib/birdsong/tweet.rb

@@ -9,16 +9,10 @@ module Birdsong
       # Check that the ids are at least real ids
       ids.each { |id| raise Birdsong::InvalidIdError if !/\A\d+\z/.match(id) }
 
-      response = self.retrieve_data_v2(ids)
-      raise Birdsong::AuthorizationError, "Invalid response code #{response.code}" unless response.code == 200
+      tweet_objects = ids.map { |id| Birdsong::TweetScraper.new.parse(id) }
 
-      json_response = JSON.parse(response.body)
-      check_for_errors(json_response)
-
-      return [] if json_response["data"].nil?
-
-      json_response["data"].map do |json_tweet|
-        Tweet.new(json_tweet, json_response["includes"])
+      tweet_objects.map do |tweet_object|
+        Tweet.new(tweet_object)
       end
     end
 
@@ -33,194 +27,26 @@ module Birdsong
     attr_reader :image_file_names
     attr_reader :video_file_names
     attr_reader :video_file_type
+    attr_reader :video_preview_image
 
     alias_method :user, :author # Every other gem uses `user` so we can just alias it
 
   private
 
-    def initialize(json_tweet, includes)
-      @json = json_tweet
-      parse(json_tweet, includes)
-    end
-
-    def parse(json_tweet, includes)
-      @id = json_tweet["id"]
-      @created_at = DateTime.parse(json_tweet["created_at"])
-      @text = json_tweet["text"]
-      @language = json_tweet["lang"]
-      @author_id = json_tweet["author_id"]
-
-      # A sanity check to make sure we have media in there correctly
-      if includes.has_key? "media"
-        media_items = includes["media"].filter do |media_item|
-          json_tweet["attachments"]["media_keys"].include? media_item["media_key"]
-        end
-      else
-        media_items = []
-      end
-
-      @image_file_names = media_items.filter_map do |media_item|
-        next unless media_item["type"] == "photo"
-        Birdsong.retrieve_media(media_item["url"])
-      end
-
-      @video_file_names = media_items.filter_map do |media_item|
-        next unless (media_item["type"] == "video") || (media_item["type"] == "animated_gif")
-
-        # If the media is video we need to fall back to V1 of the API since V2 doesn't support
-        # videos yet. This is dumb, but not a big deal.
-        media_url = get_media_url_from_extended_entities
-        media_preview_url = get_media_preview_url_from_extended_entities
-        @video_file_type = media_item["type"]
-
-        # We're returning an array because, in the case that someday more videos are available our
-        # implementations won't breaks
-        [{ url: Birdsong.retrieve_media(media_url), preview_url: Birdsong.retrieve_media(media_preview_url) }]
-      end
-
+    def initialize(tweet_object)
+      @id = tweet_object[:id]
+      @created_at = DateTime.parse(tweet_object[:date])
+      @text = tweet_object[:text]
+      @language = tweet_object[:language]
+      @author_id = tweet_object[:user][:id]
+
+      @image_file_names = tweet_object[:images]
+      @video_file_names = tweet_object[:video]
+      @video_file_type = tweet_object[:video_file_type]
+      @video_preview_image = tweet_object[:video_preview_image]
       # Look up the author given the new id.
       # NOTE: This doesn't *seem* like the right place for this, but I"m not sure where else
-      @author = User.lookup(@author_id).first
-    end
-
-    # Used to extract a GIF or video URL from the extended entities object in the Twiter API response
-    # Assumes (as is the case right now) that a Tweet cannot have more than one GIF/video
-    def get_media_url_from_extended_entities
-      response = Tweet.retrieve_data_v1(@id)
-      response = JSON.parse(response.body)
-      get_largest_variant_url(response["extended_entities"]["media"])
-    end
-
-    # Used to extract a GIF or video preview URL from the extended entities object in the Twiter API response
-    # Assumes (as is the case right now) that a Tweet cannot have more than one GIF/video
-    def get_media_preview_url_from_extended_entities
-      response = Tweet.retrieve_data_v1(@id)
-      response = JSON.parse(response.body)
-      response["extended_entities"]["media"].first["media_url_https"]
-    end
-
-    def get_largest_variant_url(media_items)
-      # The API response is pretty deeply nested, but this handles that structure
-      largest_bitrate_variant = nil
-      media_items.each do |media_item|
-        # The API returns multiple different resolutions usually. Since we only want to archive
-        # the largest we'll run through and find it
-        media_item["video_info"]["variants"].each do |variant|
-          # Usually there's constant bitrate variants, and sometimes, a .m3u playlist which is for
-          # streaming. We want to ignore that one here.
-          next unless variant&.keys.include?("bitrate")
-
-          if largest_bitrate_variant.nil? || largest_bitrate_variant["bitrate"] < variant["bitrate"]
-            largest_bitrate_variant = variant
-          end
-        end
-      end
-      largest_bitrate_variant["url"]
-    end
-
-    def self.retrieve_data_v2(ids)
-      bearer_token = Birdsong.twitter_bearer_token
-
-      tweet_lookup_url = "https://api.twitter.com/2/tweets"
-
-      # Specify the Tweet IDs that you want to lookup below (to 100 per request)
-      tweet_ids = ids.join(",")
-
-      # Add or remove optional parameters values from the params object below. Full list of parameters and their values can be found in the docs:
-      # https://developer.twitter.com/en/docs/twitter-api/tweets/lookup/api-reference
-      params = {
-        "ids": tweet_ids,
-        "expansions": "attachments.media_keys,author_id,referenced_tweets.id",
-        "tweet.fields": Birdsong.tweet_fields,
-        "user.fields": Birdsong.user_fields,
-        "media.fields": "duration_ms,height,media_key,preview_image_url,public_metrics,type,url,width",
-        "place.fields": "country_code",
-        "poll.fields": "options"
-      }
-
-      response = tweet_lookup_v2(tweet_lookup_url, bearer_token, params)
-      raise Birdsong::AuthorizationError, "Invalid response code #{response.code}" unless response.code === 200
-
-      response
-    end
-
-    def self.tweet_lookup_v2(url, bearer_token, params)
-      options = {
-        method: "get",
-        headers: {
-          "User-Agent": "v2TweetLookupRuby",
-          "Authorization": "Bearer #{bearer_token}"
-        },
-        params: params
-      }
-
-      request = Typhoeus::Request.new(url, options)
-      response = request.run
-
-      raise Birdsong::RateLimitExceeded.new(
-        response.headers["x-rate-limit-limit"],
-        response.headers["x-rate-limit-remaining"],
-        response.headers["x-rate-limit-reset"]
-      ) if response.code === 429
-      raise Birdsong::AuthorizationError, "Invalid response code #{response.code}" unless response.code === 200
-
-      response
-    end
-
-    # Note that unlike the V2 this only supports one url at a time
-    def self.retrieve_data_v1(id)
-      bearer_token = Birdsong.twitter_bearer_token
-
-      tweet_lookup_url = "https://api.twitter.com/1.1/statuses/show.json?tweet_mode=extended&id=#{id}"
-
-      response = tweet_lookup_v1(tweet_lookup_url, bearer_token)
-      raise Birdsong::RateLimitExceeded.new(
-        response.headers["x-rate-limit-limit"],
-        response.headers["x-rate-limit-remaining"],
-        response.headers["x-rate-limit-reset"]
-      ) if response.code === 429
-      raise Birdsong::AuthorizationError, "Invalid response code #{response.code}" unless response.code === 200
-
-      response
-    end
-
-    # V2 of the Twitter API (which we use everywhere else) doesn't include videos or gifs yet,
-    # so we have to fall back to V1.
-    #
-    # There's a tracker for this at https://twittercommunity.com/t/where-would-i-find-the-direct-link-to-an-mp4-video-posted-in-v2/146933/2
-    def self.tweet_lookup_v1(url, bearer_token)
-      options = {
-        method: "get",
-        headers: {
-          "Authorization": "Bearer #{bearer_token}"
-        }
-      }
-
-      request = Typhoeus::Request.new(url, options)
-      response = request.run
-
-      raise Birdsong::RateLimitExceeded.new(
-        response.headers["x-rate-limit-limit"],
-        response.headers["x-rate-limit-remaining"],
-        response.headers["x-rate-limit-reset"]
-      ) if response.code === 429
-      raise Birdsong::AuthorizationError, "Invalid response code #{response.code}" unless response.code === 200
-
-      response
-    end
-
-
-    def self.check_for_errors(parsed_json)
-      return false unless parsed_json.key?("errors")
-      return false if parsed_json["errors"].empty?
-
-      parsed_json["errors"].each do |error|
-        # If the tweet is removed, or if the user is suspended you get an Authorization Error
-        if error["title"] == "Not Found Error" || error["title"] == "Authorization Error"
-          raise Birdsong::NoTweetFoundError, "Tweet with id #{error["value"]} not found"
-        end
-      end
-      false
+      @author = User.new(tweet_object[:user])
     end
   end
 end

data/lib/birdsong/user.rb

@@ -2,21 +2,6 @@
 
 module Birdsong
   class User
-    def self.lookup(ids = [])
-      # If a single id is passed in we make it the appropriate array
-      ids = [ids] unless ids.kind_of?(Array)
-
-      # Check that the ids are at least real ids
-      ids.each { |id| raise Birdsong::InvalidIdError if !/\A\d+\z/.match(id) }
-      self.lookup_primative(ids: ids)
-    end
-
-    def self.lookup_by_usernames(usernames = [])
-      # If a single id is passed in we make it the appropriate array
-      usernames = [usernames] unless usernames.kind_of?(Array)
-      self.lookup_primative(usernames: usernames)
-    end
-
     # Attributes for after the response is parsed from Twitter
     attr_reader :json
     attr_reader :id
@@ -37,105 +22,31 @@ module Birdsong
 
   private
 
-    def initialize(json_user)
-      @json = json_user
-      parse(json_user)
+    def initialize(user_object)
+      @json = user_object.to_json
+      parse(user_object)
     end
 
-    def parse(json_user)
-      @id = json_user["id"]
-      @name = json_user["name"]
-      @username = json_user["username"]
-      @created_at = DateTime.parse(json_user["created_at"])
-      @location = json_user["location"]
+    def parse(user_object)
+      @id = user_object[:id]
+      @name = user_object[:name]
+      @username = user_object[:screen_name]
+      @created_at = DateTime.parse(user_object[:sign_up_date])
+      @location = user_object[:location]
 
       # Removing the "normal" here gets us the full-sized image, instead of the 150x150 thumbnail
-      @profile_image_url = json_user["profile_image_url"].sub!("_normal", "")
+      @profile_image_url = user_object[:profile_image_url].sub!("_normal", "")
 
-      @description = json_user["description"]
-      @url = json_user["url"]
+      @description = user_object[:description]
+      @url = user_object[:url]
       @url = "https://www.twitter.com/#{@username}" if @url.nil?
-      @followers_count = json_user["public_metrics"]["followers_count"]
-      @following_count = json_user["public_metrics"]["following_count"]
-      @tweet_count = json_user["public_metrics"]["tweet_count"]
-      @listed_count = json_user["public_metrics"]["listed_count"]
-      @verified = json_user["verified"]
-      @profile_image_file_name = Birdsong.retrieve_media(@profile_image_url)
-    end
-
-    def self.lookup_primative(usernames: nil, ids: nil)
-      raise Birdsong::InvalidIdError if usernames.nil? && ids.nil? # can't pass in nothing
-      raise Birdsong::InvalidIdError if usernames.nil? == false && ids.nil? == false # don't pass in both
-
-      response = self.retrieve_data(ids: ids, usernames: usernames)
-
-      raise Birdsong::AuthorizationError, "Invalid response code #{response.code}" unless response.code == 200
-
-      json_response = JSON.parse(response.body)
-      return [] if json_response["data"].nil?
-
-      json_response["data"].map do |json_user|
-        User.new(json_user)
-      end
-    end
 
-    def self.retrieve_data(usernames: nil, ids: nil)
-      bearer_token = Birdsong.twitter_bearer_token
-
-      raise Birdsong::InvalidIdError if usernames.nil? && ids.nil? # can't pass in nothing
-      raise Birdsong::InvalidIdError if usernames.nil? == false && ids.nil? == false # don't pass in both
-
-      # Add or remove optional parameters values from the params object below. Full list of parameters and their values can be found in the docs:
-      # https://developer.twitter.com/en/docs/twitter-api/tweets/lookup/api-reference
-      params = {
-        "expansions": "pinned_tweet_id",
-        "tweet.fields": Birdsong.tweet_fields,
-        "user.fields": Birdsong.user_fields,
-      }
-
-      if usernames.nil? == false
-        user_lookup_url = "https://api.twitter.com/2/users/by"
-        # Specify the Usernames that you want to lookup below (to 100 per request)
-        params["usernames"] = usernames.join(",")
-      elsif ids.nil? == false
-        user_lookup_url = "https://api.twitter.com/2/users"
-        # Specify the User IDs that you want to lookup below (to 100 per request)
-        params["ids"] = ids.join(",")
-      end
-
-      response = self.user_lookup(user_lookup_url, bearer_token, params)
-
-      raise Birdsong::RateLimitExceeded.new(
-        response.headers["x-rate-limit-limit"],
-        response.headers["x-rate-limit-remaining"],
-        response.headers["x-rate-limit-reset"]
-      ) if response.code === 429
-      raise Birdsong::AuthorizationError, "Invalid response code #{response.code}" unless response.code == 200
-
-      response
-    end
-
-    def self.user_lookup(url, bearer_token, params)
-      options = {
-        method: "get",
-        headers: {
-          "User-Agent": "v2UserLookupRuby",
-          "Authorization": "Bearer #{bearer_token}"
-        },
-        params: params
-      }
-
-      request = Typhoeus::Request.new(url, options)
-      response = request.run
-
-      raise Birdsong::RateLimitExceeded.new(
-        response.headers["x-rate-limit-limit"],
-        response.headers["x-rate-limit-remaining"],
-        response.headers["x-rate-limit-reset"]
-      ) if response.code === 429
-      raise Birdsong::AuthorizationError, "Invalid response code #{response.code}" unless response.code == 200
-
-      response
+      @followers_count = user_object[:followers_count]
+      @following_count = user_object[:friends_count]
+      @tweet_count = user_object[:statuses_count]
+      @listed_count = user_object[:listed_count]
+      @verified = user_object[:verified] # this will always be `false` but we're keeping it here for compatibility
+      @profile_image_file_name = Birdsong.retrieve_media(@profile_image_url)
     end
   end
 end

data/lib/birdsong/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Birdsong
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/lib/birdsong.rb

@@ -10,6 +10,11 @@ require "fileutils"
 require_relative "birdsong/version"
 require_relative "birdsong/tweet"
 require_relative "birdsong/user"
+require_relative "birdsong/scrapers/scraper"
+require_relative "birdsong/scrapers/tweet_scraper"
+require_relative "birdsong/twitter_guest_key"
+
+require_relative "birdsong/monkeypatch"
 
 module Birdsong
   extend Configuration

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: birdsong
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Christopher Guess
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-23 00:00:00.000000000 Z
+date: 2023-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: typhoeus
@@ -197,7 +197,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: A gem to interface with Twitter's API V2