birdsong 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a632c977970e4e7b014373f08dc819091aece69e59292295711ddfce261ded9c
-  data.tar.gz: ee9d3a05780af53011a3f9018c512a2f6e6dfe733824c2a0401c873b9b529188
+  metadata.gz: 9db624502354ba7be08fbe804ff22d1b1bc474fde29ff179126b0781ff42d95a
+  data.tar.gz: bfea616431da77fb69d1fa5587a5e2d7b39d6ca9ef2c09bc81adade2af7de451
 SHA512:
-  metadata.gz: 5ad9eff7a7a99201eed5821aa897dd232733510cb088e074c56a31102329a751766498800ca62a62c5327d05797e976114e173a685acc35743c82c0ca644d0cc
-  data.tar.gz: f930c1ff617cdfb255771510a1eb3fd4588dc84c0f513f8d74900c62bb7c7680047238c7ba31345f6ddbb8154fef90467be179e32db3f899d76fb335141aaa95
+  metadata.gz: 2bb4cc4b1ace6b587290aa3952fe97c024939f1ef03e3c903297fb262eb1c4719f99db288c8d7519a14b26a902eb52fc4190fc3cda0a96fac8f8622666da8eab
+  data.tar.gz: d9f5c15d5d2c0577146641785d21041264126b0827a2150a2a94ef9513b72ebbb942f1c62b037e7bb1269143e908dcd8b1bc8c7e40e363496b9d2912cc2dcc0f

data/CHANGELOG.md

@@ -3,3 +3,7 @@
 ## [0.1.0] - 2021-04-27
 
 - Initial release
+
+## [0.2.0] - 2023-10-04
+
+- Fixed to use Selenium for scraping instead of the now defunct API

data/lib/birdsong/tweet.rb

@@ -9,13 +9,10 @@ module Birdsong
       # Check that the ids are at least real ids
       ids.each { |id| raise Birdsong::InvalidIdError if !/\A\d+\z/.match(id) }
 
-      response = ids.map { |id| self.retrieve_data_v1(id) }
+      tweet_objects = ids.map { |id| Birdsong::TweetScraper.new.parse(id) }
 
-      json_response = response.map { |r| JSON.parse(r.body) }
-      check_for_errors(json_response)
-
-      json_response.map do |json_tweet|
-        Tweet.new(json_tweet)
+      tweet_objects.map do |tweet_object|
+        Tweet.new(tweet_object)
       end
     end
 
@@ -30,139 +27,26 @@ module Birdsong
     attr_reader :image_file_names
     attr_reader :video_file_names
     attr_reader :video_file_type
+    attr_reader :video_preview_image
 
     alias_method :user, :author # Every other gem uses `user` so we can just alias it
 
   private
 
-    def initialize(json_tweet)
-      @json = json_tweet
-      parse(json_tweet)
-    end
-
-    def parse(json_tweet)
-      @id = json_tweet["id"].to_s
-      @created_at = DateTime.parse(json_tweet["created_at"])
-      @text = json_tweet["full_text"]
-      @language = json_tweet["lang"]
-      @author_id = json_tweet["user"]["id"]
-
-      # A sanity check to make sure we have media in there correctly
-      if json_tweet["extended_entities"]&.has_key?("media")
-        media_items = json_tweet["extended_entities"]["media"]
-      else
-        media_items = []
-      end
-
-      @image_file_names = media_items.filter_map do |media_item|
-        next unless media_item["type"] == "photo"
-        Birdsong.retrieve_media(media_item["url"])
-      end
-
-      @video_file_names = media_items.filter_map do |media_item|
-        next unless (media_item["type"] == "video") || (media_item["type"] == "animated_gif")
-
-        # If the media is video we need to fall back to V1 of the API since V2 doesn't support
-        # videos yet. This is dumb, but not a big deal.
-        media_url = get_largest_variant_url(media_items)
-        media_preview_url = media_items.first["media_url_https"]
-        @video_file_type = media_item["type"]
-
-        # We're returning an array because, in the case that someday more videos are available our
-        # implementations won't breaks
-        [{ url: Birdsong.retrieve_media(media_url), preview_url: Birdsong.retrieve_media(media_preview_url) }]
-      end
-
+    def initialize(tweet_object)
+      @id = tweet_object[:id]
+      @created_at = DateTime.parse(tweet_object[:date])
+      @text = tweet_object[:text]
+      @language = tweet_object[:language]
+      @author_id = tweet_object[:user][:id]
+
+      @image_file_names = tweet_object[:images]
+      @video_file_names = tweet_object[:video]
+      @video_file_type = tweet_object[:video_file_type]
+      @video_preview_image = tweet_object[:video_preview_image]
       # Look up the author given the new id.
       # NOTE: This doesn't *seem* like the right place for this, but I"m not sure where else
-      @author = User.lookup(@author_id.to_s).first
-    end
-
-    def get_largest_variant_url(media_items)
-      # The API response is pretty deeply nested, but this handles that structure
-      largest_bitrate_variant = nil
-      media_items.each do |media_item|
-        # The API returns multiple different resolutions usually. Since we only want to archive
-        # the largest we'll run through and find it
-        media_item["video_info"]["variants"].each do |variant|
-          # Usually there's constant bitrate variants, and sometimes, a .m3u playlist which is for
-          # streaming. We want to ignore that one here.
-          next unless variant&.keys.include?("bitrate")
-
-          if largest_bitrate_variant.nil? || largest_bitrate_variant["bitrate"] < variant["bitrate"]
-            largest_bitrate_variant = variant
-          end
-        end
-      end
-      largest_bitrate_variant["url"]
-    end
-
-    # Note that unlike the V2 this only supports one url at a time
-    def self.retrieve_data_v1(id)
-      bearer_token = Birdsong.twitter_bearer_token
-
-      tweet_lookup_url = "https://api.twitter.com/1.1/statuses/show.json?tweet_mode=extended&id=#{id}"
-
-      response = tweet_lookup_v1(tweet_lookup_url, bearer_token)
-      raise Birdsong::RateLimitExceeded.new(
-        response.headers["x-rate-limit-limit"],
-        response.headers["x-rate-limit-remaining"],
-        response.headers["x-rate-limit-reset"]
-      ) if response.code === 429
-      raise Birdsong::AuthorizationError, "Invalid response code #{response.code}" unless response.code === 200
-
-      response
-    end
-
-    # V2 of the Twitter API (which we use everywhere else) doesn't include videos or gifs yet,
-    # so we have to fall back to V1.
-    #
-    # There's a tracker for this at https://twittercommunity.com/t/where-would-i-find-the-direct-link-to-an-mp4-video-posted-in-v2/146933/2
-    def self.tweet_lookup_v1(url, bearer_token)
-      options = {
-        method: "get",
-        headers: {
-          "Authorization": "Bearer #{bearer_token}"
-        }
-      }
-
-      request = Typhoeus::Request.new(url, options)
-      response = request.run
-
-      raise Birdsong::RateLimitExceeded.new(
-        response.headers["x-rate-limit-limit"],
-        response.headers["x-rate-limit-remaining"],
-        response.headers["x-rate-limit-reset"]
-      ) if response.code === 429
-
-      raise Birdsong::NoTweetFoundError, "Tweet with id #{url} not found" if response.code === 404
-      if response.code === 403
-        json = JSON.parse(response.body)
-        if json.has_key?("errors")
-          json["errors"].each do |error|
-            raise Birdsong::NoTweetFoundError, "User with id #{url} suspended" if error["code"] == 63
-          end
-        end
-      end
-      raise Birdsong::AuthorizationError, "Invalid response code #{response.code}" unless response.code === 200
-
-      response
-    end
-
-
-    def self.check_for_errors(parsed_json)
-      parsed_json.each do |json|
-        next unless json.key?("errors")
-        next if json["errors"].empty?
-
-        json["errors"].each do |error|
-          # If the tweet is removed, or if the user is suspended you get an Authorization Error
-          if error["title"] == "Not Found Error" || error["title"] == "Authorization Error"
-            raise Birdsong::NoTweetFoundError, "Tweet with id #{error["value"]} not found"
-          end
-        end
-      end
-      false
+      @author = User.new(tweet_object[:user])
     end
   end
 end

data/lib/birdsong/user.rb

@@ -2,21 +2,6 @@
 
 module Birdsong
   class User
-    def self.lookup(ids = [])
-      # If a single id is passed in we make it the appropriate array
-      ids = [ids] unless ids.kind_of?(Array)
-
-      # Check that the ids are at least real ids
-      ids.each { |id| raise Birdsong::InvalidIdError if !/\A\d+\z/.match(id) }
-      self.lookup_primative(ids: ids)
-    end
-
-    def self.lookup_by_usernames(usernames = [])
-      # If a single id is passed in we make it the appropriate array
-      usernames = [usernames] unless usernames.kind_of?(Array)
-      self.lookup_primative(usernames: usernames)
-    end
-
     # Attributes for after the response is parsed from Twitter
     attr_reader :json
     attr_reader :id
@@ -37,102 +22,31 @@ module Birdsong
 
   private
 
-    def initialize(json_user)
-      @json = json_user
-      parse(json_user)
+    def initialize(user_object)
+      @json = user_object.to_json
+      parse(user_object)
     end
 
-    def parse(json_user)
-      @id = json_user["id"].to_s
-      @name = json_user["name"]
-      @username = json_user["screen_name"]
-      @created_at = DateTime.parse(json_user["created_at"])
-      @location = json_user["location"]
+    def parse(user_object)
+      @id = user_object[:id]
+      @name = user_object[:name]
+      @username = user_object[:screen_name]
+      @created_at = DateTime.parse(user_object[:sign_up_date])
+      @location = user_object[:location]
 
       # Removing the "normal" here gets us the full-sized image, instead of the 150x150 thumbnail
-      @profile_image_url = json_user["profile_image_url"].sub!("_normal", "")
+      @profile_image_url = user_object[:profile_image_url].sub!("_normal", "")
 
-      @description = json_user["description"]
-      @url = json_user["url"]
+      @description = user_object[:description]
+      @url = user_object[:url]
       @url = "https://www.twitter.com/#{@username}" if @url.nil?
 
-      @followers_count = json_user["followers_count"]
-      @following_count = json_user["friends_count"]
-      @tweet_count = json_user["statuses_count"]
-      @listed_count = json_user["listed_count"]
-      @verified = json_user["verified"] # this will always be `false` but we're keeping it here for compatibility
+      @followers_count = user_object[:followers_count]
+      @following_count = user_object[:friends_count]
+      @tweet_count = user_object[:statuses_count]
+      @listed_count = user_object[:listed_count]
+      @verified = user_object[:verified] # this will always be `false` but we're keeping it here for compatibility
       @profile_image_file_name = Birdsong.retrieve_media(@profile_image_url)
     end
-
-    def self.lookup_primative(usernames: [], ids: [])
-      raise Birdsong::InvalidIdError if usernames.empty? && ids.empty? # can't pass in nothing
-
-      if usernames.empty? == false
-        response = usernames.map { |username| self.retrieve_data(username: username) }
-      elsif ids.empty? == false
-        response = ids.map { |id| self.retrieve_data(id: id) }
-      else
-        raise Birdsong::InvalidIdError
-      end
-
-      json_response = response.map { |r| JSON.parse(r.body) }
-
-      json_response.map do |json_user|
-        User.new(json_user)
-      end
-    end
-
-    def self.retrieve_data(username: nil, id: nil)
-      bearer_token = Birdsong.twitter_bearer_token
-
-      raise Birdsong::InvalidIdError if username.nil? && id.nil? # can't pass in nothing
-      raise Birdsong::InvalidIdError if username.nil? == false && id.nil? == false # don't pass in both
-
-      user_lookup_url = "https://api.twitter.com/1.1/users/show.json"
-
-      params = {}
-      if username.nil? == false
-        # Specify the Usernames that you want to lookup below (to 100 per request)
-        params["screen_name"] = username
-      elsif id.nil? == false
-        # Specify the User IDs that you want to lookup below (to 100 per request)
-        params["user_id"] = id
-      end
-
-      response = self.user_lookup(user_lookup_url, bearer_token, params)
-
-      raise Birdsong::RateLimitExceeded.new(
-        response.headers["x-rate-limit-limit"],
-        response.headers["x-rate-limit-remaining"],
-        response.headers["x-rate-limit-reset"]
-      ) if response.code === 429
-      raise Birdsong::NoTweetFoundError, "User with id #{id} or username #{username} not found" if response.code === 404
-      raise Birdsong::AuthorizationError, "Invalid response code #{response.code}" unless response.code == 200
-
-      response
-    end
-
-    def self.user_lookup(url, bearer_token, params)
-      options = {
-        method: "get",
-        headers: {
-          "User-Agent": "v2UserLookupRuby",
-          "Authorization": "Bearer #{bearer_token}"
-        },
-        params: params
-      }
-
-      request = Typhoeus::Request.new(url, options)
-      response = request.run
-
-      raise Birdsong::RateLimitExceeded.new(
-        response.headers["x-rate-limit-limit"],
-        response.headers["x-rate-limit-remaining"],
-        response.headers["x-rate-limit-reset"]
-      ) if response.code === 429
-      raise Birdsong::AuthorizationError, "Invalid response code #{response.code}" unless response.code == 200
-
-      response
-    end
   end
 end

data/lib/birdsong/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Birdsong
-  VERSION = "0.1.2"
+  VERSION = "0.2.0"
 end

data/lib/birdsong.rb

@@ -10,6 +10,11 @@ require "fileutils"
 require_relative "birdsong/version"
 require_relative "birdsong/tweet"
 require_relative "birdsong/user"
+require_relative "birdsong/scrapers/scraper"
+require_relative "birdsong/scrapers/tweet_scraper"
+require_relative "birdsong/twitter_guest_key"
+
+require_relative "birdsong/monkeypatch"
 
 module Birdsong
   extend Configuration

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: birdsong
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Christopher Guess
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-07-21 00:00:00.000000000 Z
+date: 2023-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: typhoeus
@@ -197,7 +197,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.14
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: A gem to interface with Twitter's API V2