forki 0.2.4 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4868fd61b98809521249982e057f69d69018e8defba25792382f44e06b1921a
-  data.tar.gz: 1c700275aa5fafc19e4f6b42845e5d52c02e2bc3e6cd3f1082e1a59c5c4f472d
+  metadata.gz: 78986561738c2e71c7504b8d4810c790e1cefa212b97358eae17accf4b1c2131
+  data.tar.gz: 9f72cf4a6496e4c40f3d47d566c2ec9fd1096edf8e54257223a6b26d96f0c9b2
 SHA512:
-  metadata.gz: b5967db4c8e10b9d626767f041b2f59627170a49ceb4dc47e6da11f439a668bc656550e9b0c6a3a6a52180c8e75637ab2cbfb8a3c1b007bb91c4a5280e21aabd
-  data.tar.gz: 8230ec7b913fca196e255820066efab1c520e42ac38818652cb252966a16b8ce915f5f1224606cd5eddb81968fcdbd9f37506e028c641bcad2a402536174d321
+  metadata.gz: a6fd3e3328b1c1c17d8d4bcab492588d88d7bcfc6feb20f1e85c5b75dfacc24b75c519d388353d733bf2ef0c6899f9c1804f82736d9bdb28971f21299159fed2
+  data.tar.gz: eb1f2608844ba87fc294d38091b6327de25e35f505ac1e873f5a56f797994de57fdf2a1e80d4781821330ff05cee15bc4f0d0fbe2f7a1b72ab7abab6806f7765

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    forki (0.2.1)
+    forki (0.2.5)
       apparition
       capybara
       oj
@@ -29,12 +29,13 @@ GEM
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-    addressable (2.8.4)
+    addressable (2.8.6)
       public_suffix (>= 2.0.2, < 6.0)
     apparition (0.6.0)
       capybara (~> 3.13, < 4)
       websocket-driver (>= 0.6.5)
     ast (2.4.2)
+    bigdecimal (3.1.5)
     builder (3.2.4)
     byebug (11.1.3)
     capybara (3.39.2)
@@ -53,7 +54,7 @@ GEM
     erubi (1.12.0)
     ethon (0.16.0)
       ffi (>= 1.15.0)
-    ffi (1.15.5)
+    ffi (1.16.3)
     i18n (1.13.0)
       concurrent-ruby (~> 1.0)
     json (2.6.3)
@@ -62,15 +63,16 @@ GEM
       nokogiri (>= 1.12.0)
     matrix (0.4.2)
     method_source (1.0.0)
-    mini_mime (1.1.2)
+    mini_mime (1.1.5)
     minitest (5.18.0)
     nokogiri (1.15.1-arm64-darwin)
       racc (~> 1.4)
-    oj (3.15.1)
+    oj (3.16.3)
+      bigdecimal (>= 3.0)
     parallel (1.23.0)
     parser (3.2.2.1)
       ast (~> 2.4.1)
-    public_suffix (5.0.3)
+    public_suffix (5.0.4)
     racc (1.6.2)
     rack (2.2.4)
     rack-test (2.1.0)
@@ -90,7 +92,7 @@ GEM
     rainbow (3.1.1)
     rake (13.0.6)
     regexp_parser (2.8.0)
-    rexml (3.2.5)
+    rexml (3.2.6)
     rubocop (1.51.0)
       json (~> 2.3)
       parallel (~> 1.10)
@@ -127,17 +129,17 @@ GEM
       rubocop-rails (~> 2.0)
     ruby-progressbar (1.13.0)
     rubyzip (2.3.2)
-    selenium-webdriver (4.11.0)
+    selenium-webdriver (4.16.0)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
     thor (1.2.2)
-    typhoeus (1.4.0)
+    typhoeus (1.4.1)
       ethon (>= 0.9.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.4.2)
-    websocket (1.2.9)
+    websocket (1.2.10)
     websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)

data/lib/forki/scrapers/post_scraper.rb

@@ -65,14 +65,23 @@ module Forki
       graphql_objects.any? do |graphql_object|  # if any GraphQL objects contain the top-level keys above, return true
         return true unless graphql_object.fetch("image", nil).nil? # so long as the associated values are not nil
         return true unless graphql_object.fetch("currMedia", nil).nil?
+        return true unless graphql_object.fetch("photo_image", nil).nil?
 
         # This is a complicated form for `web.facebook.com` posts
-
         if !graphql_object.dig("node", "comet_sections", "content", "story", "attachments").nil?
           if graphql_object["node"]["comet_sections"]["content"]["story"]["attachments"].count.positive?
             return true unless graphql_object["node"]["comet_sections"]["content"]["story"]["attachments"].first.dig("styles", "attachment", "all_subattachments", "nodes")&.first&.dig("media", "image", "uri").nil?
           end
         end
+
+        # Another weird format
+        begin
+          if !graphql_object["node"]["comet_sections"]["content"]["story"]["attachments"].empty?
+            return true unless graphql_object["node"]["comet_sections"]["content"]["story"]["attachments"].first.dig("styles", "attachment", "media", "photo_image", "uri").nil?
+          end
+        rescue StandardError
+
+        end
       end
     end
 
@@ -157,7 +166,7 @@ module Forki
       graphql_object_array = graphql_strings.map { |graphql_string| JSON.parse(graphql_string) }
 
       # Once in awhile it's really easy
-      video_objects = graphql_object_array.filter {|go| go.has_key?("video") }
+      video_objects = graphql_object_array.filter { |go| go.has_key?("video") }
 
       if VideoSieve.can_process_with_sieve?(graphql_object_array)
         # Eventually all of this complexity will be replaced with this
@@ -170,9 +179,15 @@ module Forki
       return extract_video_post_data_alternative(graphql_object_array) if story_node_object.nil?
 
       if story_node_object["comet_sections"]["content"]["story"]["attachments"].first["styles"]["attachment"].key?("media")
-        video_object = story_node_object["comet_sections"]["content"]["story"]["attachments"].first["styles"]["attachment"]["media"]["video"]
-        creation_date = video_object["publish_time"] if video_object&.has_key("publish_time")
-        creation_date = story_node_object["comet_sections"]["content"]["story"]["attachments"].first["styles"]["attachment"]["media"] if creation_date.nil?
+        media_object = story_node_object["comet_sections"]["content"]["story"]["attachments"].first["styles"]["attachment"]
+        if media_object.has_key?("video")
+          video_object = story_node_object["comet_sections"]["content"]["story"]["attachments"].first["styles"]["attachment"]["media"]["video"]
+        elsif media_object.has_key?("media") && media_object["media"].has_key?("browser_native_sd_url")
+          video_object = media_object["media"]
+        end
+
+        creation_date = video_object["publish_time"] if video_object&.has_key?("publish_time")
+        creation_date = story_node_object["comet_sections"]["content"]["story"]["attachments"].first["styles"]["attachment"]["media"]["publish_time"] if creation_date.nil?
       elsif story_node_object["comet_sections"]["content"]["story"]["attachments"].first["styles"]["attachment"].key?("style_infos")
         # For "Reels" we need a separate way to parse this
         video_object = story_node_object["comet_sections"]["content"]["story"]["attachments"].first["styles"]["attachment"]["style_infos"].first["fb_shorts_story"]["short_form_video_context"]["playback_video"]
@@ -181,8 +196,20 @@ module Forki
         raise "Unable to parse video object" if video_objects.empty?
       end
 
-      feedback_object = story_node_object["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]
-      reaction_counts = extract_reaction_counts(feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
+      begin
+        feedback_object = story_node_object["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]
+      rescue NoMethodError
+        feedback_object = story_node_object["comet_sections"]["feedback"]["story"]["comet_feed_ufi_container"]["story"]["story_ufi_container"]["story"]["feedback_context"]["feedback_target_with_context"]
+      end
+
+      if feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"].key?("cannot_see_top_custom_reactions")
+        reaction_counts = extract_reaction_counts(feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
+      else
+        reaction_counts = extract_reaction_counts(feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["top_reactions"])
+      end
+
+      feedback_object = feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]
+
       share_count_object = feedback_object.fetch("share_count", {})
 
       if story_node_object["comet_sections"]["content"]["story"]["comet_sections"].key? "message"
@@ -191,15 +218,37 @@ module Forki
         text = ""
       end
 
-      feedback_object["comment_list_renderer"]["feedback"]["comment_count"]["total_count"]
-      num_comments = feedback_object.has_key?("comment_list_renderer") ? feedback_object["comment_list_renderer"]["feedback"]["comment_count"]["total_count"] : feedback_object["comment_count"]["total_count"]
+      if feedback_object.has_key?("comment_list_renderer")
+        if feedback_object["comment_list_renderer"]["feedback"].key?("comment_count")
+          num_comments = feedback_object["comment_list_renderer"]["feedback"]["comment_count"]["total_count"]
+        else
+          num_comments = feedback_object["comment_list_renderer"]["feedback"]["total_comment_count"]
+        end
+
+        view_count = feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["video_view_count"]
+        reshare_warning = feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"]
+      elsif feedback_object.has_key?("comments_count_summary_renderer")
+        num_comments = feedback_object["comments_count_summary_renderer"]["feedback"]["comment_rendering_instance"]["comments"]["total_count"]
+
+        view_count = feedback_object["video_view_count"]
+        reshare_warning = feedback_object["should_show_reshare_warning"]
+      else
+        if feedback_object["feedback"].key?("comment_count")
+          num_comments = feedback_object["feedback"]["comment_count"]["total_count"]
+        else
+          num_comments = feedback_object["feedback"]["total_comment_count"]
+        end
+
+        view_count = feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["video_view_count"]
+        reshare_warning = feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"]
+      end
 
       post_details = {
         id: video_object["id"],
         num_comments: num_comments,
         num_shares: share_count_object.fetch("count", nil),
-        num_views: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["video_view_count"],
-        reshare_warning: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"],
+        num_views: view_count,
+        reshare_warning: reshare_warning,
         video_preview_image_url: video_object["preferred_thumbnail"]["image"]["uri"],
         video_url: video_object["browser_native_hd_url"] || video_object["browser_native_sd_url"],
         text: text,
@@ -217,12 +266,24 @@ module Forki
       sidepane_object = graphql_object_array.find { |graphql_object| graphql_object.key?("tahoe_sidepane_renderer") }
       video_object = graphql_object_array.find { |graphql_object| graphql_object.has_key?("video") }
       feedback_object = sidepane_object["tahoe_sidepane_renderer"]["video"]["feedback"]
-      reaction_counts = extract_reaction_counts(sidepane_object["tahoe_sidepane_renderer"]["video"]["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
+
+      if sidepane_object["tahoe_sidepane_renderer"]["video"]["feedback"].key?("cannot_see_top_custom_reactions")
+        reaction_counts = extract_reaction_counts(sidepane_object["tahoe_sidepane_renderer"]["video"]["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
+      else # if the video has no reactions, it will have a different structure
+        reaction_counts = extract_reaction_counts(sidepane_object["tahoe_sidepane_renderer"]["video"]["feedback"]["top_reactions"])
+      end
+
       share_count_object = feedback_object.fetch("share_count", {})
 
+      if feedback_object["comments_count_summary_renderer"]["feedback"].has_key?("comment_rendering_instance")
+        num_comments = feedback_object["comments_count_summary_renderer"]["feedback"]["comment_rendering_instance"]["comments"]["total_count"]
+      else
+        num_comments = feedback_object["comments_count_summary_renderer"]["feedback"]["total_comment_count"]
+      end
+
       post_details = {
         id: video_object["id"],
-        num_comments: feedback_object["comments_count_summary_renderer"]["feedback"]["total_comment_count"],
+        num_comments: num_comments,
         num_shares: share_count_object.fetch("count", nil),
         num_views: feedback_object["video_view_count"],
         reshare_warning: feedback_object["should_show_reshare_warning"],
@@ -243,42 +304,81 @@ module Forki
     # Extracts data from an image post by parsing GraphQL strings as seen in the video post scraper above
     def extract_image_post_data(graphql_object_array)
       # This is a weird one-off style
+
       graphql_object = graphql_object_array.find { |graphql_object| !graphql_object.dig("node", "comet_sections", "content", "story", "attachments").nil? }
-      unless graphql_object.nil? || graphql_object.count == 0
+      unless graphql_object.nil? || graphql_object.count.zero?
+        # TODO: These two branches are *super* similar, probably a lot of overlap
         attachments = graphql_object["node"]["comet_sections"]["content"]["story"]["attachments"]
 
-        reaction_counts = extract_reaction_counts(graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
+        if graphql_object["node"]["comet_sections"]["feedback"]["story"].key?("feedback_context")
+          feedback_object = graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
+        elsif graphql_object["node"]["comet_sections"]["feedback"]["story"].has_key?("comet_feed_ufi_container")
+          feedback_object = graphql_object["node"]["comet_sections"]["feedback"]["story"]["comet_feed_ufi_container"]["story"]["story_ufi_container"]["story"]["feedback_context"]["feedback_target_with_context"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
+        else
+          feedback_object = graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
+        end
+
+
+        if feedback_object.has_key?("cannot_see_top_custom_reactions")
+          reaction_counts = extract_reaction_counts(feedback_object["cannot_see_top_custom_reactions"]["top_reactions"])
+        else
+          reaction_counts = extract_reaction_counts(feedback_object["top_reactions"])
+        end
+
         id = graphql_object["node"]["post_id"]
-        num_comments = graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]["share_count"]["count"]
-        reshare_warning = graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"]
-        image_url = attachments.first["styles"]["attachment"]["all_subattachments"]["nodes"].first["media"]["image"]["uri"]
+        num_comments = feedback_object["share_count"]["count"]
+        reshare_warning = feedback_object["should_show_reshare_warning"]
+
+        if attachments.first["styles"]["attachment"].key?("all_subattachments")
+          image_url = attachments.first["styles"]["attachment"]["all_subattachments"]["nodes"].first["media"]["image"]["uri"]
+        else
+          image_url = attachments.first["styles"]["attachment"]["media"]["photo_image"]["uri"]
+        end
+
         text = graphql_object["node"]["comet_sections"]["content"]["story"]["message"]["text"]
         profile_link = graphql_object["node"]["comet_sections"]["content"]["story"]["actors"].first["url"]
         created_at = graphql_object["node"]["comet_sections"]["content"]["story"]["comet_sections"]["context_layout"]["story"]["comet_sections"]["metadata"].first["story"]["creation_time"]
         has_video = false
       else
-
         graphql_object_array.find { |graphql_object| graphql_object.key?("viewer_actor") && graphql_object.key?("display_comments") }
         curr_media_object = graphql_object_array.find { |graphql_object| graphql_object.key?("currMedia") }
         creation_story_object = graphql_object_array.find { |graphql_object| graphql_object.key?("creation_story") && graphql_object.key?("message") }
 
         feedback_object = graphql_object_array.find { |graphql_object| graphql_object.has_key?("comet_ufi_summary_and_actions_renderer") }["comet_ufi_summary_and_actions_renderer"]["feedback"]
+
+        if feedback_object.key?("top_reactions")
+          feedback_object = feedback_object
+        else
+          # POSSIBLY OUT OF DATE
+          feedback_object = feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]
+        end
+
         share_count_object = feedback_object.fetch("share_count", {})
 
         poster = creation_story_object["creation_story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]
 
-        reaction_counts = extract_reaction_counts(feedback_object["cannot_see_top_custom_reactions"]["top_reactions"])
+        if feedback_object.has_key?("cannot_see_top_custom_reactions")
+          reaction_counts = extract_reaction_counts(feedback_object["cannot_see_top_custom_reactions"]["top_reactions"])
+        else
+          reaction_counts = extract_reaction_counts(feedback_object["top_reactions"])
+        end
+
         id = curr_media_object["currMedia"]["id"],
+
         num_comments = feedback_object["comments_count_summary_renderer"]["feedback"]["total_comment_count"],
-        num_shares = share_count_object.fetch("count", nil),
-        reshare_warning = feedback_object["should_show_reshare_warning"],
-        image_url = curr_media_object["currMedia"]["image"]["uri"],
-        text = (creation_story_object["message"] || {}).fetch("text", nil),
-        profile_link = poster["url"],
-        created_at = curr_media_object["currMedia"]["created_time"],
-        has_video = false
+        if num_comments.nil? && feedback_object.has_key?("comments_count_summary_renderer")
+          num_comments = feedback_object["comments_count_summary_renderer"]["feedback"]["comment_rendering_instance"]["comments"]["total_count"]
+        end
 
+        num_shares = share_count_object.fetch("count", nil)
+        reshare_warning = feedback_object["should_show_reshare_warning"]
+        image_url = curr_media_object["currMedia"]["image"]["uri"]
+        text = (creation_story_object["message"] || {}).fetch("text", nil)
+        profile_link = poster["url"]
+        created_at = curr_media_object["currMedia"]["created_time"]
+        has_video = false
       end
+
       post_details = {
         id: id,
         num_comments: num_comments,
@@ -303,7 +403,12 @@ module Forki
                                                             (graphql_string.include?("live_status")) })
       video_permalink = creation_story_object["creation_story"]["shareable"]["url"].delete("\\")
       media_object = video_object["video"]["story"]["attachments"][0]["media"]
-      reaction_counts = extract_reaction_counts(creation_story_object["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
+
+      if creation_story_object["feedback"].key?("cannot_see_top_custom_reactions")
+        reaction_counts = extract_reaction_counts(creation_story_object["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
+      else
+        reaction_counts = extract_reaction_counts(creation_story_object["feedback"]["top_reactions"])
+      end
 
       post_details = {
         id: video_object["id"],
@@ -331,7 +436,11 @@ module Forki
                                                        (graphql.include? "creation_story") })["video"]["creation_story"]
       media_object = JSON.parse(graphql_strings.find { |graphql| graphql.include? "playable_url" })["video"]["creation_story"]["attachments"][0]["media"]
       video_permalink = creation_story_object["shareable"]["url"].delete("\\")
-      reaction_counts = extract_reaction_counts(creation_story_object["feedback_context"]["feedback_target_with_context"]["cannot_see_top_custom_reactions"]["top_reactions"])
+      if creation_story_object["feedback_context"]["feedback_target_with_context"].key?("cannot_see_top_custom_reactions")
+        reaction_counts = extract_reaction_counts(creation_story_object["feedback_context"]["feedback_target_with_context"]["cannot_see_top_custom_reactions"]["top_reactions"])
+      else
+        reaction_counts = extract_reaction_counts(creation_story_object["feedback_context"]["feedback_target_with_context"]["top_reactions"])
+      end
 
       post_details = {
         id: creation_story_object["shareable"]["id"],
@@ -416,4 +525,3 @@ module Forki
 end
 
 require_relative "sieves/video_sieves/video_sieve"
-

data/lib/forki/scrapers/sieves/video_sieves/video_sieve.rb

@@ -14,7 +14,7 @@ class VideoSieve
 private
 
   def self.sieve_class_for_graphql_objects(graphql_objects)
-    sieves = [VideoSieveWatchTab, VideoSieveVideoPage, VideoSieveReel]
+    sieves = [VideoSieveWatchTab, VideoSieveVideoPage, VideoSieveVideoPage2, VideoSieveReel, VideoSieveReel2]
     sieves.detect { |sieve| sieve.check(graphql_objects) }
   end
 end

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_reel.rb

@@ -4,6 +4,15 @@ class VideoSieveReel < VideoSieve
     video_object = self.extractor(graphql_objects)
 
     return false unless video_object.has_key?("short_form_video_context")
+
+    # In relation to video_sieve_reel_2
+    comment_count = graphql_objects.filter do |go|
+      go = go.first if go.kind_of?(Array) && !go.empty?
+      !go.dig("feedback", "total_comment_count").nil?
+    end.first
+
+    return false unless comment_count.nil?
+
     true
   rescue StandardError
     return false

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_reel_2.rb

@@ -0,0 +1,88 @@
+class VideoSieveReel2 < VideoSieve
+  # To check if it's valid for the inputted graphql objects
+  def self.check(graphql_objects)
+    video_object = self.extractor(graphql_objects)
+
+    return false unless video_object.has_key?("short_form_video_context")
+
+    comment_count = graphql_objects.filter do |go|
+      go = go.first if go.kind_of?(Array) && !go.empty?
+      !go.dig("feedback", "total_comment_count").nil?
+    end.first
+
+    return false if comment_count.nil?
+
+    true
+  rescue StandardError
+    return false
+  end
+
+  # output the expected format of:
+  #
+  # post_details = {
+  #   id: video_object["id"],
+  #   num_comments: num_comments,
+  #   num_shares: share_count_object.fetch("count", nil),
+  #   num_views: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["video_view_count"],
+  #   reshare_warning: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"],
+  #   video_preview_image_url: video_object["preferred_thumbnail"]["image"]["uri"],
+  #   video_url: video_object["browser_native_hd_url"] || video_object["browser_native_sd_url"],
+  #   text: text,
+  #   created_at: creation_date,
+  #   profile_link: story_node_object["comet_sections"]["context_layout"]["story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]["url"],
+  #   has_video: true
+  # }
+  # post_details[:video_preview_image_file] = Forki.retrieve_media(post_details[:video_preview_image_url])
+  # post_details[:video_file] = Forki.retrieve_media(post_details[:video_url])
+  # post_details[:reactions] = reaction_counts
+
+  def self.sieve(graphql_objects)
+    video_object = self.extractor(graphql_objects)
+
+
+    feedback_object = graphql_objects.filter do |go|
+      go = go.first if go.kind_of?(Array) && !go.empty?
+      !go.dig("feedback", "total_comment_count").nil?
+    end.first
+
+    reels_feedback_renderer = graphql_objects.filter do |go|
+      go.dig("reels_feedback_renderer")
+    end.first
+
+    reels_feedback_renderer["reels_feedback_renderer"]["story"]
+    reshare_warning = video_object["short_form_video_context"]["playback_video"].dig("warning_screen_renderer", "cix_screen", "view_model", "__typename") == "OverlayWarningScreenViewModel"
+
+    video_preview_image_url = video_object["short_form_video_context"]["playback_video"]["preferred_thumbnail"]["image"]["uri"]
+    video_url = video_object["short_form_video_context"]["playback_video"]["browser_native_hd_url"] || video_object["short_form_video_context"]["playback_video"]["browser_native_sd_url"]
+
+    post_details = {
+      id: video_object["short_form_video_context"]["video"]["id"],
+      num_comments: feedback_object["feedback"]["total_comment_count"],
+      num_shared: Forki::Scraper.extract_int_from_num_element(feedback_object["feedback"]["share_count_reduced"]),
+      num_views: nil,
+      reshare_warning: reshare_warning,
+      video_preview_image_url: video_preview_image_url,
+      video_url: video_url,
+      text: nil, # Reels don't have text
+      created_at: video_object["creation_time"],
+      profile_link: video_object["short_form_video_context"]["video_owner"]["url"],
+      has_video: true,
+      video_preview_image_file: Forki.retrieve_media(video_preview_image_url),
+      video_file: Forki.retrieve_media(video_url),
+      reactions: nil # Only available on comments it seems? Look into this again sometime
+    }
+  rescue StandardError => e
+    debugger
+  end
+
+  private
+
+  def self.extractor(graphql_objects)
+    video_objects = graphql_objects.filter do |go|
+      go = go.first if go.kind_of?(Array) && !go.empty?
+      go.has_key?("video")
+    end
+
+    video_objects.first.dig("video", "creation_story")
+  end
+end

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_video_page.rb

@@ -4,9 +4,13 @@ class VideoSieveVideoPage < VideoSieve
     story_node_object = self.extractor(graphql_objects) # This will error out
     return false unless story_node_object["content"]["story"]["attachments"].first["styles"]["attachment"].has_key?("media")
 
+    feedback_object = story_node_object["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
+    # This is what differs from video_sieve_video_page_2.rb, where this key is unnested
+    return false unless feedback_object.has_key?("cannot_see_top_custom_reactions")
+
     true
-  rescue StandardError
-    return false
+  rescue StandardError => e
+    false
   end
 
   # output the expected format of:
@@ -39,7 +43,7 @@ class VideoSieveVideoPage < VideoSieve
     video_url = video_object["browser_native_hd_url"]
     video_url = video_object["browser_native_sd_url"] if video_url.nil?
 
-    post_details = {
+    {
       id: video_object["id"],
       num_comments: feedback_object["total_comment_count"],
       num_shared: feedback_object["share_count"]["count"],

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_video_page_2.rb

@@ -0,0 +1,70 @@
+class VideoSieveVideoPage2 < VideoSieve
+  # To check if it's valid for the inputted graphql objects
+  def self.check(graphql_objects)
+    story_node_object = self.extractor(graphql_objects) # This will error out
+    return false unless story_node_object["content"]["story"]["attachments"].first["styles"]["attachment"].has_key?("media")
+
+    feedback_object = story_node_object["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
+    # This is what differs from video_sieve_video_page.rb, where this key is nested further
+    return false unless feedback_object.has_key?("top_reactions")
+
+    true
+  rescue StandardError
+    false
+  end
+
+  # output the expected format of:
+  #
+  # post_details = {
+  #   id: video_object["id"],
+  #   num_comments: num_comments,
+  #   num_shares: share_count_object.fetch("count", nil),
+  #   num_views: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["video_view_count"],
+  #   reshare_warning: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"],
+  #   video_preview_image_url: video_object["preferred_thumbnail"]["image"]["uri"],
+  #   video_url: video_object["browser_native_hd_url"] || video_object["browser_native_sd_url"],
+  #   text: text,
+  #   created_at: creation_date,
+  #   profile_link: story_node_object["comet_sections"]["context_layout"]["story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]["url"],
+  #   has_video: true
+  # }
+  # post_details[:video_preview_image_file] = Forki.retrieve_media(post_details[:video_preview_image_url])
+  # post_details[:video_file] = Forki.retrieve_media(post_details[:video_url])
+  # post_details[:reactions] = reaction_counts
+
+  def self.sieve(graphql_objects)
+    extracted_text = self.extractor(graphql_objects)
+
+    story_object = extracted_text["content"]["story"]
+    video_object = extracted_text["content"]["story"]["attachments"].first["styles"]["attachment"]["media"]
+    feedback_object = extracted_text["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
+
+    video_preview_image_url = video_object["preferred_thumbnail"]["image"]["uri"]
+    video_url = video_object["browser_native_hd_url"]
+    video_url = video_object["browser_native_sd_url"] if video_url.nil?
+
+    {
+      id: video_object["id"],
+      num_comments: feedback_object["total_comment_count"],
+      num_shared: feedback_object["share_count"]["count"],
+      num_views: nil,
+      reshare_warning: feedback_object["should_show_reshare_warning"],
+      video_preview_image_url: video_preview_image_url,
+      video_url: video_url,
+      text: story_object["message"]["text"],
+      created_at: video_object["publish_time"],
+      profile_link: story_object["actors"].first["url"],
+      has_video: true,
+      video_preview_image_file: Forki.retrieve_media(video_preview_image_url),
+      video_file: Forki.retrieve_media(video_url),
+      reactions: feedback_object["top_reactions"]["edges"]
+    }
+  end
+
+  private
+
+  def self.extractor(graphql_objects)
+    story_node_object = graphql_objects.find { |graphql_object| graphql_object.key? "node" }&.fetch("node", nil) # user posted video
+    story_node_object["comet_sections"]
+  end
+end

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_watch_tab.rb

@@ -57,6 +57,12 @@ class VideoSieveWatchTab < VideoSieve
       profile_link = filtered_json["attachments"].first["media"]["creation_story"]["comet_sections"]["title"]["story"]["actors"].first["url"]
     end
 
+    if feedback_object.key?("cannot_see_top_custom_reactions")
+      reactions = feedback_object["cannot_see_top_custom_reactions"]["top_reactions"]["edges"]
+    else
+      reactions = feedback_object["top_reactions"]["edges"]
+    end
+
     post_details = {
       id: video_object.dig("shareable", "id") || video_object["attachments"].first["media"]["id"],
       num_comments: feedback_object["total_comment_count"],
@@ -71,7 +77,7 @@ class VideoSieveWatchTab < VideoSieve
       has_video: true,
       video_preview_image_file: Forki.retrieve_media(video_preview_image_url),
       video_file: Forki.retrieve_media(video_url),
-      reactions: feedback_object["cannot_see_top_custom_reactions"]["top_reactions"]["edges"]
+      reactions: reactions
     }
   end
 

data/lib/forki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Forki
-  VERSION = "0.2.4"
+  VERSION = "0.2.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forki
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.6
 platform: ruby
 authors:
 - ''
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-10-12 00:00:00.000000000 Z
+date: 2024-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -127,7 +127,9 @@ files:
 - lib/forki/scrapers/sieves/image_sieves/image_sieve.rb.rb
 - lib/forki/scrapers/sieves/video_sieves/video_sieve.rb
 - lib/forki/scrapers/sieves/video_sieves/video_sieve_reel.rb
+- lib/forki/scrapers/sieves/video_sieves/video_sieve_reel_2.rb
 - lib/forki/scrapers/sieves/video_sieves/video_sieve_video_page.rb
+- lib/forki/scrapers/sieves/video_sieves/video_sieve_video_page_2.rb
 - lib/forki/scrapers/sieves/video_sieves/video_sieve_watch_tab.rb
 - lib/forki/scrapers/user_scraper.rb
 - lib/forki/user.rb