forki 0.2.10 → 0.2.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f761b7a50faba5ef8fc0b701031e2a06cbbed257ff1b572d466dd9c31282287d
-  data.tar.gz: 7e469c3f8ccdc65bb34f1e57c1c20bcce62cd8853b5136a0c50a9e4a71cf99e7
+  metadata.gz: 5c0373d5017b66032feca78ff34014da595078bd2ea276aed8590199ba386397
+  data.tar.gz: 41f53e6c34b490d9cf7f24ad6ff43c82c8e596c789938c1a8026411a5111245b
 SHA512:
-  metadata.gz: 9d9dfc35ff002c90d4a1821a9e00cbd1b4672142f2e199a5e2417da53ef5c4b11b5fdd2c0cca93515baa79ab56c699681f8879ae80f918bd962344f3ae5053d8
-  data.tar.gz: bf34e313bcf938328d3d94dabe5c7dcc6a855543c75fef2a04afc006c9b642097f3ead0680d6dea0eb0b22626745c89bc271b357170da68b627d4c0fce07a5ac
+  metadata.gz: c9cb10df9e69bb07c1b8f2d1da2e3369ddfaddf1900ad94e5d9568c50f4045df994939f81a9d7543371ee53b8376f93d2e2e62370a69ccffb598b83362e85ac2
+  data.tar.gz: f553f6ad7d90aece2501f1ef6d0cd8c787f1ce8bc051480bdc87dab8fa43721923670b76aa3a3b1b8a6fbc46dd834dde2025ffd79dc00987ba9bcd6899bd99f4

data/Gemfile.lock

@@ -1,12 +1,13 @@
 PATH
   remote: .
   specs:
-    forki (0.2.8)
+    forki (0.2.11)
       apparition
       capybara
       oj
       selenium-webdriver
       typhoeus
+      zorki
 
 GEM
   remote: https://rubygems.org/
@@ -29,8 +30,8 @@ GEM
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-    addressable (2.8.6)
-      public_suffix (>= 2.0.2, < 6.0)
+    addressable (2.8.7)
+      public_suffix (>= 2.0.2, < 7.0)
     apparition (0.6.0)
       capybara (~> 3.13, < 4)
       websocket-driver (>= 0.6.5)
@@ -59,6 +60,7 @@ GEM
     i18n (1.13.0)
       concurrent-ruby (~> 1.0)
     json (2.6.3)
+    logger (1.6.1)
     loofah (2.21.3)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
@@ -68,12 +70,14 @@ GEM
     minitest (5.18.0)
     nokogiri (1.15.1-arm64-darwin)
       racc (~> 1.4)
-    oj (3.16.4)
+    oj (3.16.6)
       bigdecimal (>= 3.0)
+      ostruct (>= 0.2)
+    ostruct (0.6.0)
     parallel (1.23.0)
     parser (3.2.2.1)
       ast (~> 2.4.1)
-    public_suffix (5.1.1)
+    public_suffix (6.0.1)
     racc (1.6.2)
     rack (2.2.4)
     rack-test (2.1.0)
@@ -130,8 +134,11 @@ GEM
       rubocop-rails (~> 2.0)
     ruby-progressbar (1.13.0)
     rubyzip (2.3.2)
-    selenium-webdriver (4.21.1)
+    selenium-devtools (0.129.0)
+      selenium-webdriver (~> 4.2)
+    selenium-webdriver (4.24.0)
       base64 (~> 0.2)
+      logger (~> 1.4)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
@@ -141,13 +148,20 @@ GEM
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.4.2)
-    websocket (1.2.10)
+    websocket (1.2.11)
     websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     xpath (3.2.0)
       nokogiri (~> 1.8)
     zeitwerk (2.6.8)
+    zorki (0.2.6)
+      apparition
+      capybara
+      oj
+      selenium-devtools
+      selenium-webdriver (~> 4.24.0)
+      typhoeus
 
 PLATFORMS
   arm64-darwin-21

data/forki.gemspec

@@ -36,6 +36,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "typhoeus" # For making API requests
   spec.add_dependency "oj" # A faster JSON parser/loader than stdlib
   spec.add_dependency "selenium-webdriver" # Webdriver selenium
+  spec.add_dependency "zorki" # For scraping instagram pages
 
   spec.add_development_dependency "thor" # For the generator
 

data/lib/forki/scrapers/post_scraper.rb

@@ -505,7 +505,11 @@ module Forki
         # Do nothing if element not found
       end
 
-      save_screenshot("#{Forki.temp_storage_location}/facebook_screenshot_#{SecureRandom.uuid}.png")
+      begin
+        save_screenshot("#{Forki.temp_storage_location}/facebook_screenshot_#{SecureRandom.uuid}.png")
+      rescue Selenium::WebDriver::Error::TimeoutError
+        raise Net::ReadTimeout
+      end
     end
 
     # Uses GraphQL data and DOM elements to collect information about the current post
@@ -536,14 +540,20 @@ module Forki
     rescue Net::ReadTimeout => e
       puts "Time out error: #{e}"
       puts e.backtrace
+      raise Forki::RetryableError # This insures it'll eventually be retried by Hypatia
     rescue StandardError => e
       raise e
+      raise Forki::RetryableError
     ensure
       # `page` here can be broken already. In which case we want to raise an error so it's retried later
       begin
         page.quit
       rescue Curl::Err::ConnectionFailedError
-        raise Forki::RretryableError # This insures it'll eventually be retried by Hypatia
+        raise Forki::RetryableError # This insures it'll eventually be retried by Hypatia
+      rescue StandardError => e
+        puts "Error closing browser: #{e}"
+        raise e
+        # raise Forki::RetryableError
       end
     end
   end

data/lib/forki/scrapers/scraper.rb

@@ -178,10 +178,10 @@ module Forki
     # If either of those two conditions are false, raises an exception
     def validate_and_load_page(url)
       Capybara.app_host = "https://www.facebook.com"
-      facebook_hosts = ["facebook.com", "www.facebook.com", "web.facebook.com", "m.facebook.com"]
+      facebook_hosts = ["facebook.com", "www.facebook.com", "web.facebook.com", "m.facebook.com", "l.facebook.com"]
       parsed_url = URI.parse(url)
       host = parsed_url.host
-      raise Forki::InvalidUrlError unless facebook_hosts.include?(host)
+      raise Forki::InvalidUrlError.new("Invalid Facebook host: #{host}") unless facebook_hosts.include?(host)
 
       # Replace the host with a default one to prevent redirect loops that can happen
       unless parsed_url.host == "www.facebook.com"

data/lib/forki/scrapers/user_scraper.rb

@@ -1,10 +1,12 @@
 require "typhoeus"
+require "zorki"
 
+# rubocop:disable Metrics/ClassLength
 module Forki
   class UserScraper < Scraper
     # Finds and returns the number of people who like the current page
     def find_number_of_likes(profile_details_string)
-      likes_pattern = /[0-9,.KM ] likes/
+      # likes_pattern = /[0-9,.KM ] likes/
       likes_pattern = /(?<num_likes>[0-9,.KM ]+) (l|L)ikes/
       number_of_likes_match = likes_pattern.match(profile_details_string)
 
@@ -100,7 +102,14 @@ module Forki
 
     # Uses GraphQL data and DOM elements to collect information about the current user page
     def parse(url)
+      # So some reels may actually link to an instagram user?
+      if url.include?("instagram.com")
+        user = get_instagram_user(url)
+        return user
+      end
+
       validate_and_load_page(url)
+
       graphql_strings = find_graphql_data_strings(page.html)
       is_page = graphql_strings.map { |s| JSON.parse(s) }.any? { |o| o.key?("page") }
       user_details = is_page ? extract_page_details(graphql_strings) : extract_profile_details(graphql_strings)
@@ -110,5 +119,38 @@ module Forki
 
       user_details
     end
+
+    def get_instagram_user(url)
+      uri = URI(url)
+      query = uri.query
+      components = URI.decode_uri_component(query)
+      extracted_url = URI.extract(components).first
+      extracted_uri = URI(extracted_url)
+      username = extracted_uri.to_s.match(/(https:\/\/www.instagram.com\/_u\/[\w]+)/).to_s.split("/").last
+
+      page.quit # I think we need to do this so Zorki can run?
+      zorki_users = Zorki::User.lookup(username)
+      zorki_user = nil
+
+      if zorki_users.count.positive?
+        zorki_user = zorki_users.first
+      else
+        raise ContentUnavailableError
+      end
+
+
+      # Convert a zorki_user into a hash for Forki
+      {
+        name: zorki_user.name,
+        id: username,
+        number_of_followers: zorki_user.number_of_followers,
+        verified: zorki_user.verified,
+        profile: zorki_user.profile,
+        profile_link: zorki_user.profile_link,
+        profile_image_file: zorki_user.profile_image,
+        profile_image_url: zorki_user.profile_image_url,
+        number_of_likes: 0
+      }
+    end
   end
 end

data/lib/forki/user.rb

@@ -37,7 +37,7 @@ module Forki
           def scrape(urls)
             urls.map do |url|
               user_hash = Forki::UserScraper.new.parse(url)
-              User.new(user_hash)
+              User.new(user_hash) if user_hash.is_a?(Hash)
             end
           end
       end

data/lib/forki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Forki
-  VERSION = "0.2.10"
+  VERSION = "0.2.12"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forki
 version: !ruby/object:Gem::Version
-  version: 0.2.10
+  version: 0.2.12
 platform: ruby
 authors:
 - ''
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-09-12 00:00:00.000000000 Z
+date: 2024-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: zorki
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement