forki 0.2.11 → 0.2.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a7f3c7b6c1fc45dcd2c162f6ff308f3187e07c54b921739d16c37395e3574324
-  data.tar.gz: ed7537f07b914f4500a31e948b2ee1cc8c375a15a36ea971c7186a7dbd5a7935
+  metadata.gz: b6dad4ed40e17a940d54200b022b9ca9b2569f48abcc602e6b40eb6bdc994f19
+  data.tar.gz: 96495423ff46aa90a874e2969568e0eb81fe1b0aaf1df5e3e62ec9bb46738db4
 SHA512:
-  metadata.gz: ed836bcedbe80ef77780183d0bdcc9618d8449bed16c996c1413b109360b13e343d72ffaca8b4b4da3ea42788b29a895ba2eac22be4a57523dd232144500f412
-  data.tar.gz: a676e6f6cae22491bf7d7d1ad916e8e14f96832f0000c1c60b222759cfed89fb3875c67f9a02253ee19adb9f9e247190e97e4a056b921a0b6a210c24e0e37d18
+  metadata.gz: '08a5288e3fd9feb8cfbe683986a6a99a1791699e8b2335693928b16de69a5a8ebdea955e84dd17967ab95fecde04164b9c8cbdfbcb7860ff34d12e417a597ec6'
+  data.tar.gz: 000e63e56a249cb8abca7e2ae9364e9f0f4b146d1d8063f279b135d4e9eb4593b1fe78d6d3b73e2bf89810568254aebf07aa52ea93e5a6e4d9fe0cfede1d4977

data/Gemfile.lock

@@ -1,12 +1,13 @@
 PATH
   remote: .
   specs:
-    forki (0.2.8)
+    forki (0.2.11)
       apparition
       capybara
       oj
       selenium-webdriver
       typhoeus
+      zorki
 
 GEM
   remote: https://rubygems.org/
@@ -29,8 +30,8 @@ GEM
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-    addressable (2.8.6)
-      public_suffix (>= 2.0.2, < 6.0)
+    addressable (2.8.7)
+      public_suffix (>= 2.0.2, < 7.0)
     apparition (0.6.0)
       capybara (~> 3.13, < 4)
       websocket-driver (>= 0.6.5)
@@ -59,6 +60,7 @@ GEM
     i18n (1.13.0)
       concurrent-ruby (~> 1.0)
     json (2.6.3)
+    logger (1.6.1)
     loofah (2.21.3)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
@@ -68,12 +70,14 @@ GEM
     minitest (5.18.0)
     nokogiri (1.15.1-arm64-darwin)
       racc (~> 1.4)
-    oj (3.16.4)
+    oj (3.16.6)
       bigdecimal (>= 3.0)
+      ostruct (>= 0.2)
+    ostruct (0.6.0)
     parallel (1.23.0)
     parser (3.2.2.1)
       ast (~> 2.4.1)
-    public_suffix (5.1.1)
+    public_suffix (6.0.1)
     racc (1.6.2)
     rack (2.2.4)
     rack-test (2.1.0)
@@ -130,8 +134,11 @@ GEM
       rubocop-rails (~> 2.0)
     ruby-progressbar (1.13.0)
     rubyzip (2.3.2)
-    selenium-webdriver (4.21.1)
+    selenium-devtools (0.129.0)
+      selenium-webdriver (~> 4.2)
+    selenium-webdriver (4.24.0)
       base64 (~> 0.2)
+      logger (~> 1.4)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
@@ -141,13 +148,20 @@ GEM
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.4.2)
-    websocket (1.2.10)
+    websocket (1.2.11)
     websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     xpath (3.2.0)
       nokogiri (~> 1.8)
     zeitwerk (2.6.8)
+    zorki (0.2.6)
+      apparition
+      capybara
+      oj
+      selenium-devtools
+      selenium-webdriver (~> 4.24.0)
+      typhoeus
 
 PLATFORMS
   arm64-darwin-21

data/forki.gemspec

@@ -36,6 +36,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "typhoeus" # For making API requests
   spec.add_dependency "oj" # A faster JSON parser/loader than stdlib
   spec.add_dependency "selenium-webdriver" # Webdriver selenium
+  spec.add_dependency "zorki" # For scraping instagram pages
 
   spec.add_development_dependency "thor" # For the generator
 

data/lib/forki/scrapers/post_scraper.rb

@@ -38,8 +38,6 @@ module Forki
         extract_image_post_data(graphql_objects)
       else
         extract_image_post_data(graphql_objects)
-
-        #raise UnhandledContentError
       end
     end
 
@@ -324,9 +322,7 @@ module Forki
         elsif graphql_object["node"]["comet_sections"]["feedback"]["story"].dig("feedback_context")
           begin
             feedback_object = graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
-          rescue NoMethodError
-            debugger
-          end
+          rescue NoMethodError; end
         elsif graphql_object["node"]["comet_sections"]["feedback"]["story"].has_key?("comet_feed_ufi_container")
           feedback_object = graphql_object["node"]["comet_sections"]["feedback"]["story"]["comet_feed_ufi_container"]["story"]["story_ufi_container"]["story"]["feedback_context"]["feedback_target_with_context"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
         else
@@ -353,7 +349,9 @@ module Forki
           end
         end
 
-        text = graphql_object["node"]["comet_sections"]["content"]["story"]["message"]["text"]
+        text = graphql_object["node"]["comet_sections"]["content"]["story"].dig(["message", "text"])
+        text = "" if text.nil?
+
         profile_link = graphql_object["node"]["comet_sections"]["content"]["story"]["actors"].first["url"]
 
         unless graphql_object["node"]["comet_sections"].dig("content", "story", "comet_sections", "context_layout", "story", "comet_sections", "metadata").nil?
@@ -506,7 +504,9 @@ module Forki
       end
 
       begin
+        # rubocop:disable Lint/Debugger
         save_screenshot("#{Forki.temp_storage_location}/facebook_screenshot_#{SecureRandom.uuid}.png")
+        # rubocop:enable Lint/Debugger
       rescue Selenium::WebDriver::Error::TimeoutError
         raise Net::ReadTimeout
       end

data/lib/forki/scrapers/scraper.rb

@@ -178,10 +178,10 @@ module Forki
     # If either of those two conditions are false, raises an exception
     def validate_and_load_page(url)
       Capybara.app_host = "https://www.facebook.com"
-      facebook_hosts = ["facebook.com", "www.facebook.com", "web.facebook.com", "m.facebook.com"]
+      facebook_hosts = ["facebook.com", "www.facebook.com", "web.facebook.com", "m.facebook.com", "l.facebook.com"]
       parsed_url = URI.parse(url)
       host = parsed_url.host
-      raise Forki::InvalidUrlError unless facebook_hosts.include?(host)
+      raise Forki::InvalidUrlError.new("Invalid Facebook host: #{host}") unless facebook_hosts.include?(host)
 
       # Replace the host with a default one to prevent redirect loops that can happen
       unless parsed_url.host == "www.facebook.com"

data/lib/forki/scrapers/user_scraper.rb

@@ -1,10 +1,12 @@
 require "typhoeus"
+require "zorki"
 
+# rubocop:disable Metrics/ClassLength
 module Forki
   class UserScraper < Scraper
     # Finds and returns the number of people who like the current page
     def find_number_of_likes(profile_details_string)
-      likes_pattern = /[0-9,.KM ] likes/
+      # likes_pattern = /[0-9,.KM ] likes/
       likes_pattern = /(?<num_likes>[0-9,.KM ]+) (l|L)ikes/
       number_of_likes_match = likes_pattern.match(profile_details_string)
 
@@ -100,7 +102,14 @@ module Forki
 
     # Uses GraphQL data and DOM elements to collect information about the current user page
     def parse(url)
+      # So some reels may actually link to an instagram user?
+      if url.include?("instagram.com")
+        user = get_instagram_user(url)
+        return user
+      end
+
       validate_and_load_page(url)
+
       graphql_strings = find_graphql_data_strings(page.html)
       is_page = graphql_strings.map { |s| JSON.parse(s) }.any? { |o| o.key?("page") }
       user_details = is_page ? extract_page_details(graphql_strings) : extract_profile_details(graphql_strings)
@@ -110,5 +119,38 @@ module Forki
 
       user_details
     end
+
+    def get_instagram_user(url)
+      uri = URI(url)
+      query = uri.query
+      components = URI.decode_uri_component(query)
+      extracted_url = URI.extract(components).first
+      extracted_uri = URI(extracted_url)
+      username = extracted_uri.to_s.match(/(https:\/\/www.instagram.com\/_u\/[\w]+)/).to_s.split("/").last
+
+      page.quit # I think we need to do this so Zorki can run?
+      zorki_users = Zorki::User.lookup(username)
+      zorki_user = nil
+
+      if zorki_users.count.positive?
+        zorki_user = zorki_users.first
+      else
+        raise ContentUnavailableError
+      end
+
+
+      # Convert a zorki_user into a hash for Forki
+      {
+        name: zorki_user.name,
+        id: username,
+        number_of_followers: zorki_user.number_of_followers,
+        verified: zorki_user.verified,
+        profile: zorki_user.profile,
+        profile_link: zorki_user.profile_link,
+        profile_image_file: zorki_user.profile_image,
+        profile_image_url: zorki_user.profile_image_url,
+        number_of_likes: 0
+      }
+    end
   end
 end

data/lib/forki/user.rb

@@ -37,7 +37,7 @@ module Forki
           def scrape(urls)
             urls.map do |url|
               user_hash = Forki::UserScraper.new.parse(url)
-              User.new(user_hash)
+              User.new(user_hash) if user_hash.is_a?(Hash)
             end
           end
       end

data/lib/forki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Forki
-  VERSION = "0.2.11"
+  VERSION = "0.2.13"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forki
 version: !ruby/object:Gem::Version
-  version: 0.2.11
+  version: 0.2.13
 platform: ruby
 authors:
 - ''
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-08 00:00:00.000000000 Z
+date: 2024-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: zorki
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement