birdsong 0.2.6 → 0.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82bf946c745782199f6ec1dc9e1dd21bcf2b15584e69152ffebd1b5b8b452873
-  data.tar.gz: 82dc7534653c03bd75b7f24ac588fa6c9d0fa689552f4b67e4739c296df90692
+  metadata.gz: dda08c757d1c68ec0444b27d413dd2f19691c08c4a2bfed33b99e4029e00ec81
+  data.tar.gz: 6f67bc01f6bd34abac6db6fc2b182d1f063890e44e127a8e9ecbeffc45b5db4a
 SHA512:
-  metadata.gz: 0207ae04fb995778f6fe447f55c78f962aac6c0d7418912d9dc4bf5fd575742915ba9912360811e7921655b824a3bf9ef6739a67981ec209698e112e57c9aee4
-  data.tar.gz: a8c301f6805c88ac99ef3b5306ef0c3df9ea6f710498a3d4b970c813519606e7c7cbc70b4250c8cde148f3ff50e912c7b0fad0b2dccecd0006f313bd04e299df
+  metadata.gz: 579e935b51eb486d037c5ea7a9bc75c23cd7ee3365483aa69dbe4fa2ba56f763ea0566e00eea7402635e483553028e03553a7d44d2117f7f28c4721e308bd54b
+  data.tar.gz: 81466c8e53b847b3044b1c517ba72647ceedff0857bc291830dce42c914b8261cf143cef5b00629cf80008a5af12ceef7345c88576e4345a0826339677eb9541

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    birdsong (0.2.5)
+    birdsong (0.2.8)
       capybara (~> 3.40)
       curb (~> 1.0, >= 1.0.5)
       oauth (~> 0.5.6)
@@ -13,30 +13,23 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (7.0.1)
-      actionview (= 7.0.1)
-      activesupport (= 7.0.1)
-      rack (~> 2.0, >= 2.2.0)
-      rack-test (>= 0.6.3)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actionview (7.0.1)
-      activesupport (= 7.0.1)
-      builder (~> 3.1)
-      erubi (~> 1.4)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activesupport (7.0.1)
+    activesupport (7.1.4)
+      base64
+      bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
+      mutex_m
       tzinfo (~> 2.0)
-    addressable (2.8.6)
-      public_suffix (>= 2.0.2, < 6.0)
+    addressable (2.8.7)
+      public_suffix (>= 2.0.2, < 7.0)
     ast (2.4.2)
+    backport (1.2.0)
     base64 (0.2.0)
-    bigdecimal (3.1.7)
-    builder (3.2.4)
+    benchmark (0.3.0)
+    bigdecimal (3.1.8)
     byebug (11.1.3)
     capybara (3.40.0)
       addressable
@@ -47,105 +40,131 @@ GEM
       rack-test (>= 0.6.3)
       regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
-    concurrent-ruby (1.1.9)
-    crass (1.0.6)
-    curb (1.0.5)
-    dotenv (2.7.6)
-    erubi (1.10.0)
+    concurrent-ruby (1.3.4)
+    connection_pool (2.4.1)
+    curb (1.0.6)
+    diff-lcs (1.5.1)
+    dotenv (3.1.2)
+    drb (2.2.1)
+    e2mmap (0.1.0)
     ethon (0.16.0)
       ffi (>= 1.15.0)
-    ffi (1.16.3)
-    i18n (1.9.1)
+    ffi (1.17.0-arm64-darwin)
+    ffi (1.17.0-x86_64-darwin)
+    ffi (1.17.0-x86_64-linux-gnu)
+    i18n (1.14.6)
       concurrent-ruby (~> 1.0)
-    loofah (2.13.0)
-      crass (~> 1.0.2)
-      nokogiri (>= 1.5.9)
+    jaro_winkler (1.6.0)
+    json (2.7.2)
+    kramdown (2.4.0)
+      rexml
+    kramdown-parser-gfm (1.1.0)
+      kramdown (~> 2.0)
+    language_server-protocol (3.17.0.3)
+    logger (1.6.1)
     matrix (0.4.2)
-    method_source (1.0.0)
     mini_mime (1.1.5)
-    minitest (5.15.0)
-    nokogiri (1.13.1-arm64-darwin)
+    minitest (5.25.1)
+    mutex_m (0.2.0)
+    nokogiri (1.16.7-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.13.1-x86_64-darwin)
+    nokogiri (1.16.7-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.13.1-x86_64-linux)
+    nokogiri (1.16.7-x86_64-linux)
       racc (~> 1.4)
     oauth (0.5.14)
-    oj (3.16.3)
+    oj (3.16.6)
       bigdecimal (>= 3.0)
-    parallel (1.21.0)
-    parser (3.1.0.0)
+      ostruct (>= 0.2)
+    ostruct (0.6.0)
+    parallel (1.26.3)
+    parser (3.3.5.0)
       ast (~> 2.4.1)
-    public_suffix (5.0.5)
-    racc (1.6.0)
-    rack (2.2.3)
-    rack-test (1.1.0)
-      rack (>= 1.0, < 3)
-    rails-dom-testing (2.0.3)
-      activesupport (>= 4.2.0)
-      nokogiri (>= 1.6)
-    rails-html-sanitizer (1.4.2)
-      loofah (~> 2.3)
-    railties (7.0.1)
-      actionpack (= 7.0.1)
-      activesupport (= 7.0.1)
-      method_source
-      rake (>= 12.2)
-      thor (~> 1.0)
-      zeitwerk (~> 2.5)
+      racc
+    public_suffix (6.0.1)
+    racc (1.8.1)
+    rack (3.1.7)
+    rack-test (2.1.0)
+      rack (>= 1.3)
     rainbow (3.1.1)
-    rake (13.0.6)
-    regexp_parser (2.2.0)
-    rexml (3.2.5)
-    rubocop (1.25.1)
+    rake (13.2.1)
+    rbs (2.8.4)
+    regexp_parser (2.9.2)
+    reverse_markdown (2.1.1)
+      nokogiri
+    rexml (3.3.8)
+    rubocop (1.66.1)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.1.0.0)
+      parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml
-      rubocop-ast (>= 1.15.1, < 2.0)
+      regexp_parser (>= 2.4, < 3.0)
+      rubocop-ast (>= 1.32.2, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.15.1)
-      parser (>= 3.0.1.1)
-    rubocop-minitest (0.17.1)
-      rubocop (>= 0.90, < 2.0)
-    rubocop-packaging (0.5.1)
-      rubocop (>= 0.89, < 2.0)
-    rubocop-performance (1.13.2)
-      rubocop (>= 1.7.0, < 2.0)
-      rubocop-ast (>= 0.4.0)
-    rubocop-rails (2.9.1)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.32.3)
+      parser (>= 3.3.1.0)
+    rubocop-md (1.2.3)
+      rubocop (>= 1.45)
+    rubocop-minitest (0.36.0)
+      rubocop (>= 1.61, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+    rubocop-packaging (0.5.2)
+      rubocop (>= 1.33, < 2.0)
+    rubocop-performance (1.22.1)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+    rubocop-rails (2.26.1)
       activesupport (>= 4.2.0)
       rack (>= 1.1)
-      rubocop (>= 0.90.0, < 2.0)
-    rubocop-rails_config (1.8.1)
-      railties (>= 5.0)
-      rubocop (>= 1.19)
-      rubocop-ast (>= 1.0.1)
-      rubocop-minitest (~> 0.15)
+      rubocop (>= 1.52.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+    rubocop-rails_config (1.16.0)
+      rubocop (>= 1.57.0)
+      rubocop-ast (>= 1.26.0)
+      rubocop-md
+      rubocop-minitest (~> 0.22)
       rubocop-packaging (~> 0.5)
       rubocop-performance (~> 1.11)
       rubocop-rails (~> 2.0)
-    ruby-progressbar (1.11.0)
+    ruby-progressbar (1.13.0)
     rubyzip (2.3.2)
-    selenium-devtools (0.125.0)
+    selenium-devtools (0.129.0)
       selenium-webdriver (~> 4.2)
-    selenium-webdriver (4.21.1)
+    selenium-webdriver (4.25.0)
       base64 (~> 0.2)
+      logger (~> 1.4)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
-    thor (1.2.1)
+    solargraph (0.50.0)
+      backport (~> 1.2)
+      benchmark
+      bundler (~> 2.0)
+      diff-lcs (~> 1.4)
+      e2mmap
+      jaro_winkler (~> 1.5)
+      kramdown (~> 2.3)
+      kramdown-parser-gfm (~> 1.1)
+      parser (~> 3.0)
+      rbs (~> 2.0)
+      reverse_markdown (~> 2.0)
+      rubocop (~> 1.38)
+      thor (~> 1.0)
+      tilt (~> 2.0)
+      yard (~> 0.9, >= 0.9.24)
+    thor (1.3.2)
+    tilt (2.4.0)
     typhoeus (1.4.1)
       ethon (>= 0.9.0)
-    tzinfo (2.0.4)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.1.0)
-    websocket (1.2.10)
+    unicode-display_width (2.6.0)
+    websocket (1.2.11)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.5.4)
+    yard (0.9.37)
 
 PLATFORMS
   arm64-darwin-21
@@ -164,6 +183,7 @@ DEPENDENCIES
   rubocop-performance
   rubocop-rails
   rubocop-rails_config
+  solargraph
 
 BUNDLED WITH
    2.3.26

data/birdsong.gemspec

@@ -47,6 +47,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rubocop-rails_config"
   spec.add_development_dependency "rubocop-performance"
   spec.add_development_dependency "dotenv"
+  spec.add_development_dependency "solargraph"
 
   # For more information and examples about making a new gem, checkout our
   # guide at: https://bundler.io/guides/creating_gem.html

data/lib/birdsong/scrapers/scraper.rb

@@ -67,14 +67,11 @@ module Birdsong
 
       page.driver.browser.intercept do |request, &continue|
         # This passes the request forward unmodified, since we only care about the response
-        # puts "checking request: #{request.url}"
-
         continue.call(request) && next unless request.url.include?(subpage_search)
 
-
         continue.call(request) do |response|
-
           # Check if not a CORS prefetch and finish up if not
+          # puts "checking request: #{request.url}"
           if !response.body.empty? && response.body
             check_passed = true
             unless additional_search_parameters.nil?
@@ -110,6 +107,7 @@ module Birdsong
 
       page.driver.execute_script("window.stop();")
 
+      # debugger if response_body.nil?
       raise Birdsong::NoTweetFoundError if response_body.nil?
       Oj.load(response_body)
     rescue Birdsong::WebDriverError
@@ -143,49 +141,65 @@ module Birdsong
       Capybara.current_driver = :selenium
     end
 
-    # def login
-    #   # Reset the sessions so that there's nothing laying around
-    #   page.quit
+    def login
+      # Reset the sessions so that there's nothing laying around
+      page.quit
 
-    #   # Check if we're on a Instagram page already, if not visit it.
-    #   unless page.driver.browser.current_url.include? "twitter.com"
-    #     # There seems to be a bug in the Linux ARM64 version of chromedriver where this will properly
-    #     # navigate but then timeout, crashing it all up. So instead we check and raise the error when
-    #     # that then fails again.
-    #     page.driver.browser.navigate.to("https://twitter.com")
-    #   end
+      # Check if we're on a Twitter page already, if not visit it.
+      unless page.driver.browser.current_url.include?("twitter.com") || page.driver.browser.current_url.include?("x.com")
+        # There seems to be a bug in the Linux ARM64 version of chromedriver where this will properly
+        # navigate but then timeout, crashing it all up. So instead we check and raise the error when
+        # that then fails again.
+        page.driver.browser.navigate.to("https://x.com")
+      end
 
-      # # We don't have to login if we already are
-      # begin
-      #   return if find_field("Search", wait: 10).present?
-      # rescue Capybara::ElementNotFound; end
+      # We don't have to login if we already are
+      begin
+        return if find_field("Search", wait: 10).present?
+      rescue Capybara::ElementNotFound; end
+
+      page.driver.browser.find_element(link_text: "Sign in").click      # Check if we're redirected to a login page, if we aren't we're already logged in
 
-      # # Check if we're redirected to a login page, if we aren't we're already logged in
       # return unless page.has_xpath?('//*[@id="loginForm"]/div/div[3]/button')
 
-      # # Try to log in
-      # loop_count = 0
-      # while loop_count < 5 do
-      #   fill_in("username", with: ENV["TWITTER_USER_NAME"])
-      #   fill_in("password", with: ENV["TWITTER_PASSWORD"])
-
-      #   begin
-      #     click_button("Log in", exact_text: true) # Note: "Log in" (lowercase `in`) instead redirects to Facebook's login page
-      #   rescue Capybara::ElementNotFound; end # If we can't find it don't break horribly, just keep waiting
-
-      #   break unless has_css?('p[data-testid="login-error-message"', wait: 10)
-      #   loop_count += 1
-      #   sleep(rand * 10.3)
-      # end
-
-      # # Sometimes Instagram just... doesn't let you log in
-      # raise "Instagram not accessible" if loop_count == 5
-
-      # # No we don't want to save our login credentials
-      # begin
-      #   click_on("Save Info")
-      # rescue Capybara::ElementNotFound; end
-    # end
+      # Try to log in
+      loop_count = 0
+      while loop_count < 5 do
+        3.times do
+          sleep(rand * 8.8)
+          element = page.driver.browser.find_element(tag_name: "input", name: "text")
+          next if element.nil?
+          element.click
+          break
+        rescue StandardError => e
+          puts e
+          next
+        end
+
+        sleep(rand * 2.8)
+        fill_in("text", with: ENV["TWITTER_USER_NAME"])
+        sleep(rand * 2.8)
+        find_button("Next").click
+        sleep(rand * 2.1)
+        fill_in("password", with: ENV["TWITTER_PASSWORD"])
+
+        begin
+          click_button("Log in", exact_text: true) # Note: "Log in" (lowercase `in`) instead redirects to Facebook's login page
+        rescue Capybara::ElementNotFound; end # If we can't find it don't break horribly, just keep waiting
+
+        break unless has_css?('p[data-testid="login-error-message"', wait: 10)
+        loop_count += 1
+        sleep(rand * 10.3)
+      end
+
+      # Sometimes Twitter just... doesn't let you log in
+      raise "Twitter not accessible" if loop_count == 5
+
+      # No we don't want to save our login credentials
+      begin
+        click_on("Save Info")
+      rescue Capybara::ElementNotFound; end
+    end
 
     def fetch_image(url)
       request = Typhoeus::Request.new(url, followlocation: true)

data/lib/birdsong/scrapers/tweet_scraper.rb

@@ -14,21 +14,35 @@ module Birdsong
       # - Number of likes *
       # - Hashtags
 
-      Capybara.app_host = "https://twitter.com"
+      Capybara.app_host = "https://x.com"
 
       # video slideshows https://www.instagram.com/p/CY7KxwYOFBS/?utm_source=ig_embed&utm_campaign=loading
       # login
       graphql_object = get_content_of_subpage_from_url(
-        "https://twitter.com/jack/status/#{id}",
-        "/graphql",
+        "https://x.com/jack/status/#{id}",
+        "/TweetResultByRestId",
         "data,tweetResult,result"
       )
 
       graphql_object = graphql_object.first if graphql_object.kind_of?(Array)
       graphql_object = graphql_object["data"]["tweetResult"]["result"]
 
+      raise Birdsong::NoTweetFoundError if graphql_object.nil?
+
       if graphql_object.key?("__typename") && graphql_object["__typename"] == "TweetUnavailable"
-        raise Birdsong::NoTweetFoundError
+        raise Birdsong::NoTweetFoundError if graphql_object["reason"] != "NsfwLoggedOut"
+        @@logger.info "Post is tagged NSFW, logging in to access..."
+        # Let's login and start this over?
+        login
+        @@logger.info "Logged in, retrying post..."
+
+        graphql_object = get_content_of_subpage_from_url(
+          "https://x.com/jack/status/#{id}",
+          "/TweetDetail"
+        )
+
+        # The format gets weird for this request
+        graphql_object = graphql_object["data"]["threaded_conversation_with_injections_v2"]["instructions"][0]["entries"][0]["content"]["itemContent"]["tweet_results"]["result"]["tweet"]
       end
 
       text = graphql_object["legacy"]["full_text"]
@@ -51,7 +65,7 @@ module Birdsong
             video_preview_image = Birdsong.retrieve_media(media["media_url_https"])
             video_variants = media["video_info"]["variants"]
             largest_bitrate_variant = video_variants.sort_by do |variant|
-              variant["bitrate"].nil? ? 0 : variant["bitrate"]
+              variant.has_key?("bitrate") ? variant["bitrate"] : 0
             end.last
 
             videos << Birdsong.retrieve_media(largest_bitrate_variant["url"])
@@ -106,7 +120,9 @@ module Birdsong
       # First check if a post has a fact check overlay, if so, clear it.
       # The only issue is that this can take *awhile* to search. Not sure what to do about that
       # since it's Instagram's fault for having such a fucked up obfuscated hierarchy      # Take the screenshot and return it
+      # rubocop:disable Lint/Debugger
       save_screenshot("#{Birdsong.temp_storage_location}/instagram_screenshot_#{SecureRandom.uuid}.png")
+      # rubocop:enable Lint/Debugger
     end
   end
 end

data/lib/birdsong/user.rb

@@ -39,7 +39,7 @@ module Birdsong
 
       @description = user_object[:description]
       @url = user_object[:url]
-      @url = "https://www.twitter.com/#{@username}" if @url.nil?
+      @url = "https://www.x.com/#{@username}" if @url.nil?
 
       @followers_count = user_object[:followers_count]
       @following_count = user_object[:following_count]

data/lib/birdsong/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Birdsong
-  VERSION = "0.2.6"
+  VERSION = "0.2.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: birdsong
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.8
 platform: ruby
 authors:
 - Christopher Guess
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-06-14 00:00:00.000000000 Z
+date: 2024-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: typhoeus
@@ -238,6 +238,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: solargraph
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description:
 email:
 - cguess@gmail.com
@@ -248,7 +262,6 @@ files:
 - ".github/workflows/main.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".ruby-version"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -288,7 +301,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.33
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: A gem to interface with Twitter's API V2

data/.ruby-version

@@ -1 +0,0 @@
-3.0.5