forki 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0736d43f45a908cbae8d8685a9e91b0863fc635a0544065fa3afb23d4664b64e
-  data.tar.gz: 1954cd17cb5997446cdc8040f36f0cd531c32184581f5cde20ae5783c495664d
+  metadata.gz: a0667614af238aeb8089c1af60794918c0986b7f5e12fa6fe96d33a2a5c1a06f
+  data.tar.gz: 5668a5a6056bdf9bdd9c9fd8f119fa5ed754c5ee0b152d2764b26e9f9d7a8804
 SHA512:
-  metadata.gz: 0ee46a3fe486e66f8d91a9cc025443f8c2fd98644cc2eb97effee03358ba7ca27976e2ec3d18af14d4678091c7d46a7005ed1ffe1e7744be4b783efaa0cbac58
-  data.tar.gz: f94fdbec137832b79ff52c6a63c2a2bb1d777a629df5ec474b11d18ee85335c5fd31f83d2d02dbbe31996f23e3cbb5570bd2582c98c6779744acdd39d45e98ee
+  metadata.gz: 509ffb30dada2666236ed0900e1def8a61413ee3ba2e8705e31bd01422f8d2baf2cd24479faf906b6374602fcb0c1dc7bb334b7e02a560c798eba6efab30a2c9
+  data.tar.gz: 3f79b804b5505222e06a49352807d92c95f13a8735bedcd1b32cbf079be72b0922052014ef82da8771db108f7b0b674cd132caf7d377084c57cf0d245eb049e1

data/lib/forki/scrapers/post_scraper.rb

@@ -62,8 +62,16 @@ module Forki
 
     def check_if_post_is_image(graphql_objects)
       graphql_objects.any? do |graphql_object|  # if any GraphQL objects contain the top-level keys above, return true
-        true unless graphql_object.fetch("image", nil).nil? # so long as the associated values are not nil
-        true unless graphql_object.fetch("currMedia", nil).nil?
+        return true unless graphql_object.fetch("image", nil).nil? # so long as the associated values are not nil
+        return true unless graphql_object.fetch("currMedia", nil).nil?
+
+        # This is a complicated form for `web.facebook.com` posts
+
+        if !graphql_object.dig("node", "comet_sections", "content", "story", "attachments").nil?
+          if graphql_object["node"]["comet_sections"]["content"]["story"]["attachments"].count.positive?
+            return true unless graphql_object["node"]["comet_sections"]["content"]["story"]["attachments"].first.dig("styles", "attachment", "all_subattachments", "nodes")&.first&.dig("media", "image", "uri").nil?
+          end
+        end
       end
     end
 
@@ -220,26 +228,53 @@ module Forki
 
     # Extracts data from an image post by parsing GraphQL strings as seen in the video post scraper above
     def extract_image_post_data(graphql_object_array)
-      graphql_object_array.find { |graphql_object| graphql_object.key?("viewer_actor") && graphql_object.key?("display_comments") }
-      curr_media_object = graphql_object_array.find { |graphql_object| graphql_object.key?("currMedia") }
-      creation_story_object = graphql_object_array.find { |graphql_object| graphql_object.key?("creation_story") && graphql_object.key?("message") }
+      # This is a weird one-off style
+      graphql_object = graphql_object_array.find { |graphql_object| !graphql_object.dig("node", "comet_sections", "content", "story", "attachments").nil? }
+      unless graphql_object.nil? || graphql_object.count == 0
+        attachments = graphql_object["node"]["comet_sections"]["content"]["story"]["attachments"]
+
+        reaction_counts = extract_reaction_counts(graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
+        id = graphql_object["node"]["post_id"]
+        num_comments = graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]["share_count"]["count"]
+        reshare_warning = graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"]
+        image_url = attachments.first["styles"]["attachment"]["all_subattachments"]["nodes"].first["media"]["image"]["uri"]
+        text = graphql_object["node"]["comet_sections"]["content"]["story"]["message"]["text"]
+        profile_link = graphql_object["node"]["comet_sections"]["content"]["story"]["actors"].first["url"]
+        created_at = graphql_object["node"]["comet_sections"]["content"]["story"]["comet_sections"]["context_layout"]["story"]["comet_sections"]["metadata"].first["story"]["creation_time"]
+        has_video = false
+      else
 
-      feedback_object = graphql_object_array.find { |graphql_object| graphql_object.has_key?("comet_ufi_summary_and_actions_renderer") }["comet_ufi_summary_and_actions_renderer"]["feedback"]
-      share_count_object = feedback_object.fetch("share_count", {})
+        graphql_object_array.find { |graphql_object| graphql_object.key?("viewer_actor") && graphql_object.key?("display_comments") }
+        curr_media_object = graphql_object_array.find { |graphql_object| graphql_object.key?("currMedia") }
+        creation_story_object = graphql_object_array.find { |graphql_object| graphql_object.key?("creation_story") && graphql_object.key?("message") }
+
+        feedback_object = graphql_object_array.find { |graphql_object| graphql_object.has_key?("comet_ufi_summary_and_actions_renderer") }["comet_ufi_summary_and_actions_renderer"]["feedback"]
+        share_count_object = feedback_object.fetch("share_count", {})
+
+        poster = creation_story_object["creation_story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]
 
-      poster = creation_story_object["creation_story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]
+        reaction_counts = extract_reaction_counts(feedback_object["cannot_see_top_custom_reactions"]["top_reactions"])
+        id = curr_media_object["currMedia"]["id"],
+        num_comments = feedback_object["comments_count_summary_renderer"]["feedback"]["total_comment_count"],
+        num_shares = share_count_object.fetch("count", nil),
+        reshare_warning = feedback_object["should_show_reshare_warning"],
+        image_url = curr_media_object["currMedia"]["image"]["uri"],
+        text = (creation_story_object["message"] || {}).fetch("text", nil),
+        profile_link = poster["url"],
+        created_at = curr_media_object["currMedia"]["created_time"],
+        has_video = false
 
-      reaction_counts = extract_reaction_counts(feedback_object["cannot_see_top_custom_reactions"]["top_reactions"])
+      end
       post_details = {
-        id: curr_media_object["currMedia"]["id"],
-        num_comments: feedback_object["comments_count_summary_renderer"]["feedback"]["total_comment_count"],
-        num_shares: share_count_object.fetch("count", nil),
-        reshare_warning: feedback_object["should_show_reshare_warning"],
-        image_url: curr_media_object["currMedia"]["image"]["uri"],
-        text: (creation_story_object["message"] || {}).fetch("text", nil),
-        profile_link: poster["url"],
-        created_at: curr_media_object["currMedia"]["created_time"],
-        has_video: false
+        id: id,
+        num_comments: num_comments,
+        num_shares: num_shares,
+        reshare_warning: reshare_warning,
+        image_url: image_url,
+        text: text,
+        profile_link: profile_link,
+        created_at: created_at,
+        has_video: has_video
       }
       post_details[:image_file] = Forki.retrieve_media(post_details[:image_url])
       post_details[:reactions] = reaction_counts

data/lib/forki/scrapers/scraper.rb

@@ -6,7 +6,7 @@ require "dotenv/load"
 require "oj"
 require "selenium-webdriver"
 require "open-uri"
-require 'selenium/webdriver/remote/http/curb'
+require "selenium/webdriver/remote/http/curb"
 
 options = Selenium::WebDriver::Options.chrome(exclude_switches: ["enable-automation"])
 options.add_argument("--start-maximized")

data/lib/forki/scrapers/user_scraper.rb

@@ -39,9 +39,14 @@ module Forki
         profile_title_section = graphql_strings.find { |gql| gql.include? "profile_tile_section_type" }
 
         json = JSON.parse(profile_title_section)
-        followers_node = json["user"]["profile_tile_sections"]["edges"].first["node"]["profile_tile_views"]["nodes"][1]["view_style_renderer"]["view"]["profile_tile_items"]["nodes"].select do |node|
-          node["node"]["timeline_context_item"]["timeline_context_list_item_type"] == "INTRO_CARD_FOLLOWERS"
-        end
+
+        followers_node = []
+        begin
+          followers_node = json["user"]["profile_tile_sections"]["edges"].first["node"]["profile_tile_views"]["nodes"][1]["view_style_renderer"]["view"]["profile_tile_items"]["nodes"].select do |node|
+            node["node"]["timeline_context_item"]["timeline_context_list_item_type"] == "INTRO_CARD_FOLLOWERS"
+          end
+        rescue NoMethodError; end
+
         if followers_node.empty?
           number_of_followers = nil
         else

data/lib/forki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Forki
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forki
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - ''
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-27 00:00:00.000000000 Z
+date: 2023-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara