forki 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb9eb80820ccf86f9a99d1c757940a935137d834a2bd76d855deaeee470840a1
-  data.tar.gz: 667f0600afdf53ced62b57def3021a6ccc0cf2a194142e2f23beb76c8384a35a
+  metadata.gz: a0667614af238aeb8089c1af60794918c0986b7f5e12fa6fe96d33a2a5c1a06f
+  data.tar.gz: 5668a5a6056bdf9bdd9c9fd8f119fa5ed754c5ee0b152d2764b26e9f9d7a8804
 SHA512:
-  metadata.gz: 78aa5d9bf07a1f5790f8e1705056956b30a7d2854723b74fdfd0a7612be2c3988916333bfc5fc15dff62d62fda1d01452f375765b4e1ddc6633148372993ebbb
-  data.tar.gz: 21d6c81749a1f1fe160508e9efe53ca6abc2be59b01eb31684a500df79b9092647de47d1545ea21c0c701c5d1e34f2f40c35dcf0765bdfce6d7c72ec60feee2b
+  metadata.gz: 509ffb30dada2666236ed0900e1def8a61413ee3ba2e8705e31bd01422f8d2baf2cd24479faf906b6374602fcb0c1dc7bb334b7e02a560c798eba6efab30a2c9
+  data.tar.gz: 3f79b804b5505222e06a49352807d92c95f13a8735bedcd1b32cbf079be72b0922052014ef82da8771db108f7b0b674cd132caf7d377084c57cf0d245eb049e1

data/Gemfile

@@ -19,3 +19,5 @@ gem "dotenv", "~> 2.7.6"
 gem "byebug"
 
 gem "rack", "2.2.4"
+
+gem "curb", "~> 1.0", ">= 1.0.5"

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    forki (0.1.0)
+    forki (0.1.1)
       apparition
       capybara
       oj
@@ -48,6 +48,7 @@ GEM
       xpath (~> 3.2)
     concurrent-ruby (1.2.2)
     crass (1.0.6)
+    curb (1.0.5)
     dotenv (2.7.6)
     erubi (1.12.0)
     ethon (0.16.0)
@@ -150,6 +151,7 @@ PLATFORMS
 
 DEPENDENCIES
   byebug
+  curb (~> 1.0, >= 1.0.5)
   dotenv (~> 2.7.6)
   forki!
   minitest (~> 5.0)

data/lib/forki/scrapers/post_scraper.rb

@@ -62,8 +62,16 @@ module Forki
 
     def check_if_post_is_image(graphql_objects)
       graphql_objects.any? do |graphql_object|  # if any GraphQL objects contain the top-level keys above, return true
-        true unless graphql_object.fetch("image", nil).nil? # so long as the associated values are not nil
-        true unless graphql_object.fetch("currMedia", nil).nil?
+        return true unless graphql_object.fetch("image", nil).nil? # so long as the associated values are not nil
+        return true unless graphql_object.fetch("currMedia", nil).nil?
+
+        # This is a complicated form for `web.facebook.com` posts
+
+        if !graphql_object.dig("node", "comet_sections", "content", "story", "attachments").nil?
+          if graphql_object["node"]["comet_sections"]["content"]["story"]["attachments"].count.positive?
+            return true unless graphql_object["node"]["comet_sections"]["content"]["story"]["attachments"].first.dig("styles", "attachment", "all_subattachments", "nodes")&.first&.dig("media", "image", "uri").nil?
+          end
+        end
       end
     end
 
@@ -220,26 +228,53 @@ module Forki
 
     # Extracts data from an image post by parsing GraphQL strings as seen in the video post scraper above
     def extract_image_post_data(graphql_object_array)
-      graphql_object_array.find { |graphql_object| graphql_object.key?("viewer_actor") && graphql_object.key?("display_comments") }
-      curr_media_object = graphql_object_array.find { |graphql_object| graphql_object.key?("currMedia") }
-      creation_story_object = graphql_object_array.find { |graphql_object| graphql_object.key?("creation_story") && graphql_object.key?("message") }
+      # This is a weird one-off style
+      graphql_object = graphql_object_array.find { |graphql_object| !graphql_object.dig("node", "comet_sections", "content", "story", "attachments").nil? }
+      unless graphql_object.nil? || graphql_object.count == 0
+        attachments = graphql_object["node"]["comet_sections"]["content"]["story"]["attachments"]
+
+        reaction_counts = extract_reaction_counts(graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
+        id = graphql_object["node"]["post_id"]
+        num_comments = graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]["share_count"]["count"]
+        reshare_warning = graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"]
+        image_url = attachments.first["styles"]["attachment"]["all_subattachments"]["nodes"].first["media"]["image"]["uri"]
+        text = graphql_object["node"]["comet_sections"]["content"]["story"]["message"]["text"]
+        profile_link = graphql_object["node"]["comet_sections"]["content"]["story"]["actors"].first["url"]
+        created_at = graphql_object["node"]["comet_sections"]["content"]["story"]["comet_sections"]["context_layout"]["story"]["comet_sections"]["metadata"].first["story"]["creation_time"]
+        has_video = false
+      else
 
-      feedback_object = graphql_object_array.find { |graphql_object| graphql_object.has_key?("comet_ufi_summary_and_actions_renderer") }["comet_ufi_summary_and_actions_renderer"]["feedback"]
-      share_count_object = feedback_object.fetch("share_count", {})
+        graphql_object_array.find { |graphql_object| graphql_object.key?("viewer_actor") && graphql_object.key?("display_comments") }
+        curr_media_object = graphql_object_array.find { |graphql_object| graphql_object.key?("currMedia") }
+        creation_story_object = graphql_object_array.find { |graphql_object| graphql_object.key?("creation_story") && graphql_object.key?("message") }
+
+        feedback_object = graphql_object_array.find { |graphql_object| graphql_object.has_key?("comet_ufi_summary_and_actions_renderer") }["comet_ufi_summary_and_actions_renderer"]["feedback"]
+        share_count_object = feedback_object.fetch("share_count", {})
+
+        poster = creation_story_object["creation_story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]
 
-      poster = creation_story_object["creation_story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]
+        reaction_counts = extract_reaction_counts(feedback_object["cannot_see_top_custom_reactions"]["top_reactions"])
+        id = curr_media_object["currMedia"]["id"],
+        num_comments = feedback_object["comments_count_summary_renderer"]["feedback"]["total_comment_count"],
+        num_shares = share_count_object.fetch("count", nil),
+        reshare_warning = feedback_object["should_show_reshare_warning"],
+        image_url = curr_media_object["currMedia"]["image"]["uri"],
+        text = (creation_story_object["message"] || {}).fetch("text", nil),
+        profile_link = poster["url"],
+        created_at = curr_media_object["currMedia"]["created_time"],
+        has_video = false
 
-      reaction_counts = extract_reaction_counts(feedback_object["cannot_see_top_custom_reactions"]["top_reactions"])
+      end
       post_details = {
-        id: curr_media_object["currMedia"]["id"],
-        num_comments: feedback_object["comments_count_summary_renderer"]["feedback"]["total_comment_count"],
-        num_shares: share_count_object.fetch("count", nil),
-        reshare_warning: feedback_object["should_show_reshare_warning"],
-        image_url: curr_media_object["currMedia"]["image"]["uri"],
-        text: (creation_story_object["message"] || {}).fetch("text", nil),
-        profile_link: poster["url"],
-        created_at: curr_media_object["currMedia"]["created_time"],
-        has_video: false
+        id: id,
+        num_comments: num_comments,
+        num_shares: num_shares,
+        reshare_warning: reshare_warning,
+        image_url: image_url,
+        text: text,
+        profile_link: profile_link,
+        created_at: created_at,
+        has_video: has_video
       }
       post_details[:image_file] = Forki.retrieve_media(post_details[:image_url])
       post_details[:reactions] = reaction_counts
@@ -349,8 +384,9 @@ module Forki
       page.quit
 
       post_data
-    rescue Net::ReadTimeout
-      # Eat it?
+    rescue Net::ReadTimeout => e
+      puts "Time out error: #{e}"
+      puts e.backtrace
     rescue StandardError => e
       raise e
     ensure

data/lib/forki/scrapers/scraper.rb

@@ -6,6 +6,7 @@ require "dotenv/load"
 require "oj"
 require "selenium-webdriver"
 require "open-uri"
+require "selenium/webdriver/remote/http/curb"
 
 options = Selenium::WebDriver::Options.chrome(exclude_switches: ["enable-automation"])
 options.add_argument("--start-maximized")
@@ -20,8 +21,8 @@ options.add_argument("--remote-debugging-port=9222")
 options.add_argument("--user-data-dir=/tmp/tarun_forki_#{SecureRandom.uuid}")
 
 Capybara.register_driver :selenium_forki do |app|
-  client = Selenium::WebDriver::Remote::Http::Default.new
-  client.read_timeout = 60  # Don't wait 60 seconds to return Net::ReadTimeoutError. We'll retry through Hypatia after 10 seconds
+  client = Selenium::WebDriver::Remote::Http::Curb.new
+  # client.read_timeout = 60  # Don't wait 60 seconds to return Net::ReadTimeoutError. We'll retry through Hypatia after 10 seconds
   Capybara::Selenium::Driver.new(app, browser: :chrome, options: options, http_client: client)
 end
 
@@ -97,8 +98,8 @@ module Forki
       options.add_argument("--user-data-dir=/tmp/tarun_forki_#{SecureRandom.uuid}")
 
       Capybara.register_driver :selenium_forki do |app|
-        client = Selenium::WebDriver::Remote::Http::Default.new
-        client.read_timeout = 60  # Don't wait 60 seconds to return Net::ReadTimeoutError. We'll retry through Hypatia after 10 seconds
+        client = Selenium::WebDriver::Remote::Http::Curb.new
+        # client.read_timeout = 60  # Don't wait 60 seconds to return Net::ReadTimeoutError. We'll retry through Hypatia after 10 seconds
         Capybara::Selenium::Driver.new(app, browser: :chrome, options: options, http_client: client)
       end
 
@@ -110,7 +111,9 @@ module Forki
       raise MissingCredentialsError if ENV["FACEBOOK_EMAIL"].nil? || ENV["FACEBOOK_PASSWORD"].nil?
 
       url ||= "https://www.facebook.com"
-      visit(url)  # Visit the url passed in or the facebook homepage if nothing is
+
+
+      page.driver.browser.navigate.to(url)  # Visit the url passed in or the facebook homepage if nothing is
 
       # Look for "login_form" box, which throws an error if not found. So we catch it and run the rest of the tests
       begin
@@ -120,7 +123,7 @@ module Forki
       end
 
       # Since we're not logged in, let's do that quick
-      visit("https://www.facebook.com") if login_form.nil?
+      page.driver.browser.navigate.to("https://www.facebook.com") if login_form.nil?
 
       login_form.fill_in("email", with: ENV["FACEBOOK_EMAIL"])
       login_form.fill_in("pass", with: ENV["FACEBOOK_PASSWORD"])

data/lib/forki/scrapers/user_scraper.rb

@@ -39,9 +39,14 @@ module Forki
         profile_title_section = graphql_strings.find { |gql| gql.include? "profile_tile_section_type" }
 
         json = JSON.parse(profile_title_section)
-        followers_node = json["user"]["profile_tile_sections"]["edges"].first["node"]["profile_tile_views"]["nodes"][1]["view_style_renderer"]["view"]["profile_tile_items"]["nodes"].select do |node|
-          node["node"]["timeline_context_item"]["timeline_context_list_item_type"] == "INTRO_CARD_FOLLOWERS"
-        end
+
+        followers_node = []
+        begin
+          followers_node = json["user"]["profile_tile_sections"]["edges"].first["node"]["profile_tile_views"]["nodes"][1]["view_style_renderer"]["view"]["profile_tile_items"]["nodes"].select do |node|
+            node["node"]["timeline_context_item"]["timeline_context_list_item_type"] == "INTRO_CARD_FOLLOWERS"
+          end
+        rescue NoMethodError; end
+
         if followers_node.empty?
           number_of_followers = nil
         else

data/lib/forki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Forki
-  VERSION = "0.1.1"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forki
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - ''
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-23 00:00:00.000000000 Z
+date: 2023-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara