birdsong 0.2.8 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dda08c757d1c68ec0444b27d413dd2f19691c08c4a2bfed33b99e4029e00ec81
-  data.tar.gz: 6f67bc01f6bd34abac6db6fc2b182d1f063890e44e127a8e9ecbeffc45b5db4a
+  metadata.gz: d9a2c2815cf9cbc7d98e308971c1924f90bba4d2715d7ded7c4e275f523ec84f
+  data.tar.gz: cb888671a2171786b5b7c68c96b4d1e0a8957a1d44ce3a1897b4f0fde43fad8b
 SHA512:
-  metadata.gz: 579e935b51eb486d037c5ea7a9bc75c23cd7ee3365483aa69dbe4fa2ba56f763ea0566e00eea7402635e483553028e03553a7d44d2117f7f28c4721e308bd54b
-  data.tar.gz: 81466c8e53b847b3044b1c517ba72647ceedff0857bc291830dce42c914b8261cf143cef5b00629cf80008a5af12ceef7345c88576e4345a0826339677eb9541
+  metadata.gz: 42d22a774b5cc9d1e2518911aaf4e3e7f84e5731d8662103d7e5df48ae71c25c0f723efaf334b60cfb3b18274cebce8119d88765a06f7b6957626506e5abcd74
+  data.tar.gz: 402eafa8cfd7873e5284ded6fce36a050ea8fa5055a240c6fe460e17bd92ea7d55637b8052bff5afd1fc110f66a16f97360c75d5b47249b25d0d974df230ea5b

data/.ruby-version

@@ -0,0 +1 @@
+3.0.5

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    birdsong (0.2.8)
+    birdsong (0.2.7)
       capybara (~> 3.40)
       curb (~> 1.0, >= 1.0.5)
       oauth (~> 0.5.6)
@@ -13,24 +13,25 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.1.4)
+    activesupport (7.1.5)
       base64
+      benchmark (>= 0.3)
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
       connection_pool (>= 2.2.5)
       drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
       mutex_m
+      securerandom (>= 0.3)
       tzinfo (~> 2.0)
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
     ast (2.4.2)
-    backport (1.2.0)
     base64 (0.2.0)
     benchmark (0.3.0)
     bigdecimal (3.1.8)
-    byebug (11.1.3)
     capybara (3.40.0)
       addressable
       matrix
@@ -43,23 +44,20 @@ GEM
     concurrent-ruby (1.3.4)
     connection_pool (2.4.1)
     curb (1.0.6)
-    diff-lcs (1.5.1)
-    dotenv (3.1.2)
+    debug (1.7.0)
+      irb (>= 1.5.0)
+      reline (>= 0.3.1)
+    dotenv (3.1.4)
     drb (2.2.1)
-    e2mmap (0.1.0)
     ethon (0.16.0)
       ffi (>= 1.15.0)
-    ffi (1.17.0-arm64-darwin)
-    ffi (1.17.0-x86_64-darwin)
-    ffi (1.17.0-x86_64-linux-gnu)
+    ffi (1.17.0)
     i18n (1.14.6)
       concurrent-ruby (~> 1.0)
-    jaro_winkler (1.6.0)
-    json (2.7.2)
-    kramdown (2.4.0)
-      rexml
-    kramdown-parser-gfm (1.1.0)
-      kramdown (~> 2.0)
+    io-console (0.6.0)
+    irb (1.6.2)
+      reline (>= 0.3.0)
+    json (2.7.5)
     language_server-protocol (3.17.0.3)
     logger (1.6.1)
     matrix (0.4.2)
@@ -78,22 +76,21 @@ GEM
       ostruct (>= 0.2)
     ostruct (0.6.0)
     parallel (1.26.3)
-    parser (3.3.5.0)
+    parser (3.3.5.1)
       ast (~> 2.4.1)
       racc
     public_suffix (6.0.1)
     racc (1.8.1)
-    rack (3.1.7)
+    rack (3.1.8)
     rack-test (2.1.0)
       rack (>= 1.3)
     rainbow (3.1.1)
     rake (13.2.1)
-    rbs (2.8.4)
     regexp_parser (2.9.2)
-    reverse_markdown (2.1.1)
-      nokogiri
-    rexml (3.3.8)
-    rubocop (1.66.1)
+    reline (0.3.2)
+      io-console (~> 0.5)
+    rexml (3.3.9)
+    rubocop (1.68.0)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
@@ -103,9 +100,9 @@ GEM
       rubocop-ast (>= 1.32.2, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.32.3)
+    rubocop-ast (1.33.0)
       parser (>= 3.3.1.0)
-    rubocop-md (1.2.3)
+    rubocop-md (1.2.4)
       rubocop (>= 1.45)
     rubocop-minitest (0.36.0)
       rubocop (>= 1.61, < 2.0)
@@ -115,7 +112,7 @@ GEM
     rubocop-performance (1.22.1)
       rubocop (>= 1.48.1, < 2.0)
       rubocop-ast (>= 1.31.1, < 2.0)
-    rubocop-rails (2.26.1)
+    rubocop-rails (2.27.0)
       activesupport (>= 4.2.0)
       rack (>= 1.1)
       rubocop (>= 1.52.0, < 2.0)
@@ -130,32 +127,15 @@ GEM
       rubocop-rails (~> 2.0)
     ruby-progressbar (1.13.0)
     rubyzip (2.3.2)
-    selenium-devtools (0.129.0)
+    securerandom (0.3.1)
+    selenium-devtools (0.130.0)
       selenium-webdriver (~> 4.2)
-    selenium-webdriver (4.25.0)
+    selenium-webdriver (4.26.0)
       base64 (~> 0.2)
       logger (~> 1.4)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
-    solargraph (0.50.0)
-      backport (~> 1.2)
-      benchmark
-      bundler (~> 2.0)
-      diff-lcs (~> 1.4)
-      e2mmap
-      jaro_winkler (~> 1.5)
-      kramdown (~> 2.3)
-      kramdown-parser-gfm (~> 1.1)
-      parser (~> 3.0)
-      rbs (~> 2.0)
-      reverse_markdown (~> 2.0)
-      rubocop (~> 1.38)
-      thor (~> 1.0)
-      tilt (~> 2.0)
-      yard (~> 0.9, >= 0.9.24)
-    thor (1.3.2)
-    tilt (2.4.0)
     typhoeus (1.4.1)
       ethon (>= 0.9.0)
     tzinfo (2.0.6)
@@ -164,7 +144,6 @@ GEM
     websocket (1.2.11)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    yard (0.9.37)
 
 PLATFORMS
   arm64-darwin-21
@@ -175,7 +154,7 @@ PLATFORMS
 
 DEPENDENCIES
   birdsong!
-  byebug
+  debug
   dotenv
   minitest
   rake
@@ -183,7 +162,6 @@ DEPENDENCIES
   rubocop-performance
   rubocop-rails
   rubocop-rails_config
-  solargraph
 
 BUNDLED WITH
    2.3.26

data/birdsong.gemspec

@@ -39,7 +39,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "selenium-devtools"
 
   # Dev dependencies
-  spec.add_development_dependency "byebug"
+  spec.add_development_dependency "debug"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest"
   spec.add_development_dependency "rubocop"
@@ -47,7 +47,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rubocop-rails_config"
   spec.add_development_dependency "rubocop-performance"
   spec.add_development_dependency "dotenv"
-  spec.add_development_dependency "solargraph"
 
   # For more information and examples about making a new gem, checkout our
   # guide at: https://bundler.io/guides/creating_gem.html

data/lib/birdsong/scrapers/scraper.rb

@@ -67,11 +67,14 @@ module Birdsong
 
       page.driver.browser.intercept do |request, &continue|
         # This passes the request forward unmodified, since we only care about the response
+        # puts "checking request: #{request.url}"
+
         continue.call(request) && next unless request.url.include?(subpage_search)
 
+
         continue.call(request) do |response|
+
           # Check if not a CORS prefetch and finish up if not
-          # puts "checking request: #{request.url}"
           if !response.body.empty? && response.body
             check_passed = true
             unless additional_search_parameters.nil?
@@ -107,7 +110,6 @@ module Birdsong
 
       page.driver.execute_script("window.stop();")
 
-      # debugger if response_body.nil?
       raise Birdsong::NoTweetFoundError if response_body.nil?
       Oj.load(response_body)
     rescue Birdsong::WebDriverError
@@ -141,65 +143,49 @@ module Birdsong
       Capybara.current_driver = :selenium
     end
 
-    def login
-      # Reset the sessions so that there's nothing laying around
-      page.quit
+    # def login
+    #   # Reset the sessions so that there's nothing laying around
+    #   page.quit
 
-      # Check if we're on a Twitter page already, if not visit it.
-      unless page.driver.browser.current_url.include?("twitter.com") || page.driver.browser.current_url.include?("x.com")
-        # There seems to be a bug in the Linux ARM64 version of chromedriver where this will properly
-        # navigate but then timeout, crashing it all up. So instead we check and raise the error when
-        # that then fails again.
-        page.driver.browser.navigate.to("https://x.com")
-      end
+    #   # Check if we're on a Instagram page already, if not visit it.
+    #   unless page.driver.browser.current_url.include? "twitter.com" || page.driver.browser.current_url.include? "x.com"
+    #     # There seems to be a bug in the Linux ARM64 version of chromedriver where this will properly
+    #     # navigate but then timeout, crashing it all up. So instead we check and raise the error when
+    #     # that then fails again.
+    #     page.driver.browser.navigate.to("https://x.com")
+    #   end
 
-      # We don't have to login if we already are
-      begin
-        return if find_field("Search", wait: 10).present?
-      rescue Capybara::ElementNotFound; end
-
-      page.driver.browser.find_element(link_text: "Sign in").click      # Check if we're redirected to a login page, if we aren't we're already logged in
+      # # We don't have to login if we already are
+      # begin
+      #   return if find_field("Search", wait: 10).present?
+      # rescue Capybara::ElementNotFound; end
 
+      # # Check if we're redirected to a login page, if we aren't we're already logged in
       # return unless page.has_xpath?('//*[@id="loginForm"]/div/div[3]/button')
 
-      # Try to log in
-      loop_count = 0
-      while loop_count < 5 do
-        3.times do
-          sleep(rand * 8.8)
-          element = page.driver.browser.find_element(tag_name: "input", name: "text")
-          next if element.nil?
-          element.click
-          break
-        rescue StandardError => e
-          puts e
-          next
-        end
-
-        sleep(rand * 2.8)
-        fill_in("text", with: ENV["TWITTER_USER_NAME"])
-        sleep(rand * 2.8)
-        find_button("Next").click
-        sleep(rand * 2.1)
-        fill_in("password", with: ENV["TWITTER_PASSWORD"])
-
-        begin
-          click_button("Log in", exact_text: true) # Note: "Log in" (lowercase `in`) instead redirects to Facebook's login page
-        rescue Capybara::ElementNotFound; end # If we can't find it don't break horribly, just keep waiting
-
-        break unless has_css?('p[data-testid="login-error-message"', wait: 10)
-        loop_count += 1
-        sleep(rand * 10.3)
-      end
-
-      # Sometimes Twitter just... doesn't let you log in
-      raise "Twitter not accessible" if loop_count == 5
-
-      # No we don't want to save our login credentials
-      begin
-        click_on("Save Info")
-      rescue Capybara::ElementNotFound; end
-    end
+      # # Try to log in
+      # loop_count = 0
+      # while loop_count < 5 do
+      #   fill_in("username", with: ENV["TWITTER_USER_NAME"])
+      #   fill_in("password", with: ENV["TWITTER_PASSWORD"])
+
+      #   begin
+      #     click_button("Log in", exact_text: true) # Note: "Log in" (lowercase `in`) instead redirects to Facebook's login page
+      #   rescue Capybara::ElementNotFound; end # If we can't find it don't break horribly, just keep waiting
+
+      #   break unless has_css?('p[data-testid="login-error-message"', wait: 10)
+      #   loop_count += 1
+      #   sleep(rand * 10.3)
+      # end
+
+      # # Sometimes Instagram just... doesn't let you log in
+      # raise "Instagram not accessible" if loop_count == 5
+
+      # # No we don't want to save our login credentials
+      # begin
+      #   click_on("Save Info")
+      # rescue Capybara::ElementNotFound; end
+    # end
 
     def fetch_image(url)
       request = Typhoeus::Request.new(url, followlocation: true)

data/lib/birdsong/scrapers/tweet_scraper.rb

@@ -2,6 +2,7 @@
 
 require "typhoeus"
 require_relative "scraper"
+require "debug"
 
 module Birdsong
   class TweetScraper < Scraper
@@ -20,31 +21,20 @@ module Birdsong
       # login
       graphql_object = get_content_of_subpage_from_url(
         "https://x.com/jack/status/#{id}",
-        "/TweetResultByRestId",
+        "/graphql",
         "data,tweetResult,result"
       )
 
       graphql_object = graphql_object.first if graphql_object.kind_of?(Array)
       graphql_object = graphql_object["data"]["tweetResult"]["result"]
 
-      raise Birdsong::NoTweetFoundError if graphql_object.nil?
-
       if graphql_object.key?("__typename") && graphql_object["__typename"] == "TweetUnavailable"
-        raise Birdsong::NoTweetFoundError if graphql_object["reason"] != "NsfwLoggedOut"
-        @@logger.info "Post is tagged NSFW, logging in to access..."
-        # Let's login and start this over?
-        login
-        @@logger.info "Logged in, retrying post..."
-
-        graphql_object = get_content_of_subpage_from_url(
-          "https://x.com/jack/status/#{id}",
-          "/TweetDetail"
-        )
-
-        # The format gets weird for this request
-        graphql_object = graphql_object["data"]["threaded_conversation_with_injections_v2"]["instructions"][0]["entries"][0]["content"]["itemContent"]["tweet_results"]["result"]["tweet"]
+        raise Birdsong::NoTweetFoundError
       end
 
+      # Certain types of tweets are wrapped in a "tweet" object
+      graphql_object = graphql_object["tweet"] if graphql_object.key?("tweet")
+
       text = graphql_object["legacy"]["full_text"]
       date = graphql_object["legacy"]["created_at"]
       id   = graphql_object["legacy"]["id_str"]
@@ -65,7 +55,7 @@ module Birdsong
             video_preview_image = Birdsong.retrieve_media(media["media_url_https"])
             video_variants = media["video_info"]["variants"]
             largest_bitrate_variant = video_variants.sort_by do |variant|
-              variant.has_key?("bitrate") ? variant["bitrate"] : 0
+              variant["bitrate"].nil? ? 0 : variant["bitrate"]
             end.last
 
             videos << Birdsong.retrieve_media(largest_bitrate_variant["url"])
@@ -122,7 +112,7 @@ module Birdsong
       # since it's Instagram's fault for having such a fucked up obfuscated hierarchy      # Take the screenshot and return it
       # rubocop:disable Lint/Debugger
       save_screenshot("#{Birdsong.temp_storage_location}/instagram_screenshot_#{SecureRandom.uuid}.png")
-      # rubocop:enable Lint/Debugger
+      # rubocop:enable Link/Debugger
     end
   end
 end

data/lib/birdsong/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Birdsong
-  VERSION = "0.2.8"
+  VERSION = "0.2.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: birdsong
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.2.9
 platform: ruby
 authors:
 - Christopher Guess
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-21 00:00:00.000000000 Z
+date: 2024-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: typhoeus
@@ -127,7 +127,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: byebug
+  name: debug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -238,20 +238,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: solargraph
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description:
 email:
 - cguess@gmail.com
@@ -262,6 +248,7 @@ files:
 - ".github/workflows/main.yml"
 - ".gitignore"
 - ".rubocop.yml"
+- ".ruby-version"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -301,7 +288,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: A gem to interface with Twitter's API V2