forki 0.2.4 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4868fd61b98809521249982e057f69d69018e8defba25792382f44e06b1921a
-  data.tar.gz: 1c700275aa5fafc19e4f6b42845e5d52c02e2bc3e6cd3f1082e1a59c5c4f472d
+  metadata.gz: cd276bc782515b3b7935eab759da6cf2acdafc146798a56e3e21ee667358ad3c
+  data.tar.gz: 532d2c06542a0bdfe4e22f1a0b2ec3cf3ded6057b30de93386fad442bfa686ab
 SHA512:
-  metadata.gz: b5967db4c8e10b9d626767f041b2f59627170a49ceb4dc47e6da11f439a668bc656550e9b0c6a3a6a52180c8e75637ab2cbfb8a3c1b007bb91c4a5280e21aabd
-  data.tar.gz: 8230ec7b913fca196e255820066efab1c520e42ac38818652cb252966a16b8ce915f5f1224606cd5eddb81968fcdbd9f37506e028c641bcad2a402536174d321
+  metadata.gz: 46aaf4eef616f99ca44eac48134b151af41e1755b808c2efef18fdca905956d8f5cb72d54e61070f64ae883b8e64e88f5adfb94f0d41a166eae136f0474133df
+  data.tar.gz: 36c8bc7c506c952c036f38eb655522fad7a6b4cd6bf7c3de11e78e60aec4f5c3f124d5449e1aa40425a22cacee71e7c393239092a7e6dbf057a7e46e90457f1a

data/lib/forki/scrapers/post_scraper.rb

@@ -182,7 +182,12 @@ module Forki
       end
 
       feedback_object = story_node_object["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]
-      reaction_counts = extract_reaction_counts(feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
+      if feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"].key?("cannot_see_top_custom_reactions")
+        reaction_counts = extract_reaction_counts(feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
+      else
+        reaction_counts = extract_reaction_counts(feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["top_reactions"])
+      end
+
       share_count_object = feedback_object.fetch("share_count", {})
 
       if story_node_object["comet_sections"]["content"]["story"]["comet_sections"].key? "message"
@@ -191,8 +196,19 @@ module Forki
         text = ""
       end
 
-      feedback_object["comment_list_renderer"]["feedback"]["comment_count"]["total_count"]
-      num_comments = feedback_object.has_key?("comment_list_renderer") ? feedback_object["comment_list_renderer"]["feedback"]["comment_count"]["total_count"] : feedback_object["comment_count"]["total_count"]
+      if feedback_object.has_key?("comment_list_renderer")
+        if feedback_object["comment_list_renderer"]["feedback"].key?("comment_count")
+          num_comments = feedback_object["comment_list_renderer"]["feedback"]["comment_count"]["total_count"]
+        else
+          num_comments = feedback_object["comment_list_renderer"]["feedback"]["total_comment_count"]
+        end
+      else
+        if feedback_object["feedback"].key?("comment_count")
+          num_comments = feedback_object["feedback"]["comment_count"]["total_count"]
+        else
+          num_comments = feedback_object["feedback"]["total_comment_count"]
+        end
+      end
 
       post_details = {
         id: video_object["id"],
@@ -217,7 +233,13 @@ module Forki
       sidepane_object = graphql_object_array.find { |graphql_object| graphql_object.key?("tahoe_sidepane_renderer") }
       video_object = graphql_object_array.find { |graphql_object| graphql_object.has_key?("video") }
       feedback_object = sidepane_object["tahoe_sidepane_renderer"]["video"]["feedback"]
-      reaction_counts = extract_reaction_counts(sidepane_object["tahoe_sidepane_renderer"]["video"]["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
+
+      if sidepane_object["tahoe_sidepane_renderer"]["video"]["feedback"].key?("cannot_see_top_custom_reactions")
+        reaction_counts = extract_reaction_counts(sidepane_object["tahoe_sidepane_renderer"]["video"]["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
+      else # if the video has no reactions, it will have a different structure
+        reaction_counts = extract_reaction_counts(sidepane_object["tahoe_sidepane_renderer"]["video"]["feedback"]["top_reactions"])
+      end
+
       share_count_object = feedback_object.fetch("share_count", {})
 
       post_details = {
@@ -247,7 +269,12 @@ module Forki
       unless graphql_object.nil? || graphql_object.count == 0
         attachments = graphql_object["node"]["comet_sections"]["content"]["story"]["attachments"]
 
-        reaction_counts = extract_reaction_counts(graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
+        if graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"].has_key?("cannot_see_top_custom_reactions")
+          reaction_counts = extract_reaction_counts(graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
+        else
+          reaction_counts = extract_reaction_counts(graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]["top_reactions"])
+        end
+
         id = graphql_object["node"]["post_id"]
         num_comments = graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]["share_count"]["count"]
         reshare_warning = graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"]
@@ -267,7 +294,12 @@ module Forki
 
         poster = creation_story_object["creation_story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]
 
-        reaction_counts = extract_reaction_counts(feedback_object["cannot_see_top_custom_reactions"]["top_reactions"])
+        if feedback_object.has_key?("cannot_see_top_custom_reactions")
+          reaction_counts = extract_reaction_counts(feedback_object["cannot_see_top_custom_reactions"]["top_reactions"])
+        else
+          reaction_counts = extract_reaction_counts(feedback_object["top_reactions"])
+        end
+
         id = curr_media_object["currMedia"]["id"],
         num_comments = feedback_object["comments_count_summary_renderer"]["feedback"]["total_comment_count"],
         num_shares = share_count_object.fetch("count", nil),
@@ -303,7 +335,12 @@ module Forki
                                                             (graphql_string.include?("live_status")) })
       video_permalink = creation_story_object["creation_story"]["shareable"]["url"].delete("\\")
       media_object = video_object["video"]["story"]["attachments"][0]["media"]
-      reaction_counts = extract_reaction_counts(creation_story_object["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
+
+      if creation_story_object["feedback"].key?("cannot_see_top_custom_reactions")
+        reaction_counts = extract_reaction_counts(creation_story_object["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
+      else
+        reaction_counts = extract_reaction_counts(creation_story_object["feedback"]["top_reactions"])
+      end
 
       post_details = {
         id: video_object["id"],
@@ -331,7 +368,11 @@ module Forki
                                                        (graphql.include? "creation_story") })["video"]["creation_story"]
       media_object = JSON.parse(graphql_strings.find { |graphql| graphql.include? "playable_url" })["video"]["creation_story"]["attachments"][0]["media"]
       video_permalink = creation_story_object["shareable"]["url"].delete("\\")
-      reaction_counts = extract_reaction_counts(creation_story_object["feedback_context"]["feedback_target_with_context"]["cannot_see_top_custom_reactions"]["top_reactions"])
+      if creation_story_object["feedback_context"]["feedback_target_with_context"].key?("cannot_see_top_custom_reactions")
+        reaction_counts = extract_reaction_counts(creation_story_object["feedback_context"]["feedback_target_with_context"]["cannot_see_top_custom_reactions"]["top_reactions"])
+      else
+        reaction_counts = extract_reaction_counts(creation_story_object["feedback_context"]["feedback_target_with_context"]["top_reactions"])
+      end
 
       post_details = {
         id: creation_story_object["shareable"]["id"],

data/lib/forki/scrapers/sieves/video_sieves/video_sieve.rb

@@ -14,7 +14,7 @@ class VideoSieve
 private
 
   def self.sieve_class_for_graphql_objects(graphql_objects)
-    sieves = [VideoSieveWatchTab, VideoSieveVideoPage, VideoSieveReel]
+    sieves = [VideoSieveWatchTab, VideoSieveVideoPage, VideoSieveVideoPage2, VideoSieveReel, VideoSieveReel2]
     sieves.detect { |sieve| sieve.check(graphql_objects) }
   end
 end

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_reel.rb

@@ -4,6 +4,15 @@ class VideoSieveReel < VideoSieve
     video_object = self.extractor(graphql_objects)
 
     return false unless video_object.has_key?("short_form_video_context")
+
+    # In relation to video_sieve_reel_2
+    comment_count = graphql_objects.filter do |go|
+      go = go.first if go.kind_of?(Array) && !go.empty?
+      !go.dig("feedback", "total_comment_count").nil?
+    end.first
+
+    return false unless comment_count.nil?
+
     true
   rescue StandardError
     return false

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_reel_2.rb

@@ -0,0 +1,88 @@
+class VideoSieveReel2 < VideoSieve
+  # To check if it's valid for the inputted graphql objects
+  def self.check(graphql_objects)
+    video_object = self.extractor(graphql_objects)
+
+    return false unless video_object.has_key?("short_form_video_context")
+
+    comment_count = graphql_objects.filter do |go|
+      go = go.first if go.kind_of?(Array) && !go.empty?
+      !go.dig("feedback", "total_comment_count").nil?
+    end.first
+
+    return false if comment_count.nil?
+
+    true
+  rescue StandardError
+    return false
+  end
+
+  # output the expected format of:
+  #
+  # post_details = {
+  #   id: video_object["id"],
+  #   num_comments: num_comments,
+  #   num_shares: share_count_object.fetch("count", nil),
+  #   num_views: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["video_view_count"],
+  #   reshare_warning: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"],
+  #   video_preview_image_url: video_object["preferred_thumbnail"]["image"]["uri"],
+  #   video_url: video_object["browser_native_hd_url"] || video_object["browser_native_sd_url"],
+  #   text: text,
+  #   created_at: creation_date,
+  #   profile_link: story_node_object["comet_sections"]["context_layout"]["story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]["url"],
+  #   has_video: true
+  # }
+  # post_details[:video_preview_image_file] = Forki.retrieve_media(post_details[:video_preview_image_url])
+  # post_details[:video_file] = Forki.retrieve_media(post_details[:video_url])
+  # post_details[:reactions] = reaction_counts
+
+  def self.sieve(graphql_objects)
+    video_object = self.extractor(graphql_objects)
+
+
+    feedback_object = graphql_objects.filter do |go|
+      go = go.first if go.kind_of?(Array) && !go.empty?
+      !go.dig("feedback", "total_comment_count").nil?
+    end.first
+
+    reels_feedback_renderer = graphql_objects.filter do |go|
+      go.dig("reels_feedback_renderer")
+    end.first
+
+    reels_feedback_renderer["reels_feedback_renderer"]["story"]
+    reshare_warning = video_object["short_form_video_context"]["playback_video"].dig("warning_screen_renderer", "cix_screen", "view_model", "__typename") == "OverlayWarningScreenViewModel"
+
+    video_preview_image_url = video_object["short_form_video_context"]["playback_video"]["preferred_thumbnail"]["image"]["uri"]
+    video_url = video_object["short_form_video_context"]["playback_video"]["browser_native_hd_url"] || video_object["short_form_video_context"]["playback_video"]["browser_native_sd_url"]
+
+    post_details = {
+      id: video_object["short_form_video_context"]["video"]["id"],
+      num_comments: feedback_object["feedback"]["total_comment_count"],
+      num_shared: Forki::Scraper.extract_int_from_num_element(feedback_object["feedback"]["share_count_reduced"]),
+      num_views: nil,
+      reshare_warning: reshare_warning,
+      video_preview_image_url: video_preview_image_url,
+      video_url: video_url,
+      text: nil, # Reels don't have text
+      created_at: video_object["creation_time"],
+      profile_link: video_object["short_form_video_context"]["video_owner"]["url"],
+      has_video: true,
+      video_preview_image_file: Forki.retrieve_media(video_preview_image_url),
+      video_file: Forki.retrieve_media(video_url),
+      reactions: nil # Only available on comments it seems? Look into this again sometime
+    }
+  rescue StandardError => e
+    debugger
+  end
+
+  private
+
+  def self.extractor(graphql_objects)
+    video_objects = graphql_objects.filter do |go|
+      go = go.first if go.kind_of?(Array) && !go.empty?
+      go.has_key?("video")
+    end
+
+    video_objects.first.dig("video", "creation_story")
+  end
+end

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_video_page.rb

@@ -4,9 +4,13 @@ class VideoSieveVideoPage < VideoSieve
     story_node_object = self.extractor(graphql_objects) # This will error out
     return false unless story_node_object["content"]["story"]["attachments"].first["styles"]["attachment"].has_key?("media")
 
+    feedback_object = story_node_object["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
+    # This is what differs from video_sieve_video_page_2.rb, where this key is unnested
+    return false unless feedback_object.has_key?("cannot_see_top_custom_reactions")
+
     true
-  rescue StandardError
-    return false
+  rescue StandardError => e
+    false
   end
 
   # output the expected format of:
@@ -39,7 +43,7 @@ class VideoSieveVideoPage < VideoSieve
     video_url = video_object["browser_native_hd_url"]
     video_url = video_object["browser_native_sd_url"] if video_url.nil?
 
-    post_details = {
+    {
       id: video_object["id"],
       num_comments: feedback_object["total_comment_count"],
       num_shared: feedback_object["share_count"]["count"],

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_video_page_2.rb

@@ -0,0 +1,70 @@
+class VideoSieveVideoPage2 < VideoSieve
+  # To check if it's valid for the inputted graphql objects
+  def self.check(graphql_objects)
+    story_node_object = self.extractor(graphql_objects) # This will error out
+    return false unless story_node_object["content"]["story"]["attachments"].first["styles"]["attachment"].has_key?("media")
+
+    feedback_object = story_node_object["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
+    # This is what differs from video_sieve_video_page.rb, where this key is nested further
+    return false unless feedback_object.has_key?("top_reactions")
+
+    true
+  rescue StandardError
+    false
+  end
+
+  # output the expected format of:
+  #
+  # post_details = {
+  #   id: video_object["id"],
+  #   num_comments: num_comments,
+  #   num_shares: share_count_object.fetch("count", nil),
+  #   num_views: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["video_view_count"],
+  #   reshare_warning: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"],
+  #   video_preview_image_url: video_object["preferred_thumbnail"]["image"]["uri"],
+  #   video_url: video_object["browser_native_hd_url"] || video_object["browser_native_sd_url"],
+  #   text: text,
+  #   created_at: creation_date,
+  #   profile_link: story_node_object["comet_sections"]["context_layout"]["story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]["url"],
+  #   has_video: true
+  # }
+  # post_details[:video_preview_image_file] = Forki.retrieve_media(post_details[:video_preview_image_url])
+  # post_details[:video_file] = Forki.retrieve_media(post_details[:video_url])
+  # post_details[:reactions] = reaction_counts
+
+  def self.sieve(graphql_objects)
+    extracted_text = self.extractor(graphql_objects)
+
+    story_object = extracted_text["content"]["story"]
+    video_object = extracted_text["content"]["story"]["attachments"].first["styles"]["attachment"]["media"]
+    feedback_object = extracted_text["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
+
+    video_preview_image_url = video_object["preferred_thumbnail"]["image"]["uri"]
+    video_url = video_object["browser_native_hd_url"]
+    video_url = video_object["browser_native_sd_url"] if video_url.nil?
+
+    {
+      id: video_object["id"],
+      num_comments: feedback_object["total_comment_count"],
+      num_shared: feedback_object["share_count"]["count"],
+      num_views: nil,
+      reshare_warning: feedback_object["should_show_reshare_warning"],
+      video_preview_image_url: video_preview_image_url,
+      video_url: video_url,
+      text: story_object["message"]["text"],
+      created_at: video_object["publish_time"],
+      profile_link: story_object["actors"].first["url"],
+      has_video: true,
+      video_preview_image_file: Forki.retrieve_media(video_preview_image_url),
+      video_file: Forki.retrieve_media(video_url),
+      reactions: feedback_object["top_reactions"]["edges"]
+    }
+  end
+
+  private
+
+  def self.extractor(graphql_objects)
+    story_node_object = graphql_objects.find { |graphql_object| graphql_object.key? "node" }&.fetch("node", nil) # user posted video
+    story_node_object["comet_sections"]
+  end
+end

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_watch_tab.rb

@@ -57,6 +57,12 @@ class VideoSieveWatchTab < VideoSieve
       profile_link = filtered_json["attachments"].first["media"]["creation_story"]["comet_sections"]["title"]["story"]["actors"].first["url"]
     end
 
+    if feedback_object.key?("cannot_see_top_custom_reactions")
+      reactions = feedback_object["cannot_see_top_custom_reactions"]["top_reactions"]["edges"]
+    else
+      reactions = feedback_object["top_reactions"]["edges"]
+    end
+
     post_details = {
       id: video_object.dig("shareable", "id") || video_object["attachments"].first["media"]["id"],
       num_comments: feedback_object["total_comment_count"],
@@ -71,7 +77,7 @@ class VideoSieveWatchTab < VideoSieve
       has_video: true,
       video_preview_image_file: Forki.retrieve_media(video_preview_image_url),
       video_file: Forki.retrieve_media(video_url),
-      reactions: feedback_object["cannot_see_top_custom_reactions"]["top_reactions"]["edges"]
+      reactions: reactions
     }
   end
 

data/lib/forki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Forki
-  VERSION = "0.2.4"
+  VERSION = "0.2.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forki
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors:
 - ''
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-10-12 00:00:00.000000000 Z
+date: 2023-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -127,7 +127,9 @@ files:
 - lib/forki/scrapers/sieves/image_sieves/image_sieve.rb.rb
 - lib/forki/scrapers/sieves/video_sieves/video_sieve.rb
 - lib/forki/scrapers/sieves/video_sieves/video_sieve_reel.rb
+- lib/forki/scrapers/sieves/video_sieves/video_sieve_reel_2.rb
 - lib/forki/scrapers/sieves/video_sieves/video_sieve_video_page.rb
+- lib/forki/scrapers/sieves/video_sieves/video_sieve_video_page_2.rb
 - lib/forki/scrapers/sieves/video_sieves/video_sieve_watch_tab.rb
 - lib/forki/scrapers/user_scraper.rb
 - lib/forki/user.rb