forki 0.2.14 → 0.2.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8076712122eaa3930f4810048520db6fb17452584a7347a8cb2d4d53d275984a
-  data.tar.gz: 74b06da01c47fd9d4dd9532a79a7aaf03e2e2f760c9bf91cb015bfd1a82b5789
+  metadata.gz: 5a81f3f14449d4d0abdb3300bbdffbc4d34b8c5db400c8a4d7beb9bb2958fcc7
+  data.tar.gz: d4b9ade188b782c02da0b53ef64fb39a6d66aaed3eac2354dd968e87c58f1e51
 SHA512:
-  metadata.gz: b85a149035c8aa693c2d758dbf246725eb64ce6159c86ec67f29c9ba11ec99de98b987fdd71c9536fa1f097ddfff9fa3fe172dd4d6e7640b44439316f0e153d4
-  data.tar.gz: 60f2298e805c9ea0517983a2f17dfe388e62f41bff9525d2b5161ab9306f729eb7d8fb2d1f35d1c963dfd2db3bdaa93f318e546f8771780a61dd03d0c059b432
+  metadata.gz: d9516edea544bd8eefaee678e60f957f9f659a32daa46bbb8157a06da38e4e513ea3eca85108c59714b4d9d70b536ef853a7f6741e1e956c2bdb7c89a908308d
+  data.tar.gz: eb6126c73693e2d13d335d517573736343869b94e3e17e8775d65f2c496e90b55c6baed06797e140ca76539ba5434d2ad2dc603de415d758db76337ee74224c6

data/Gemfile

@@ -7,7 +7,6 @@ gemspec
 
 gem "rake", "~> 13.0"
 
-gem "minitest", "~> 5.0"
 
 gem "rubocop", "~> 1.7"
 
@@ -21,3 +20,5 @@ gem "byebug"
 gem "rack", "2.2.4"
 
 gem "curb", "~> 1.0", ">= 1.0.5"
+
+gem "minitest", "~> 5.25"

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    forki (0.2.11)
+    forki (0.2.14)
       apparition
       capybara
       oj
@@ -67,7 +67,7 @@ GEM
     matrix (0.4.2)
     method_source (1.0.0)
     mini_mime (1.1.5)
-    minitest (5.18.0)
+    minitest (5.25.1)
     nokogiri (1.15.1-arm64-darwin)
       racc (~> 1.4)
     oj (3.16.6)
@@ -97,7 +97,7 @@ GEM
     rainbow (3.1.1)
     rake (13.0.6)
     regexp_parser (2.8.0)
-    rexml (3.2.6)
+    rexml (3.3.9)
     rubocop (1.51.0)
       json (~> 2.3)
       parallel (~> 1.10)
@@ -136,7 +136,7 @@ GEM
     rubyzip (2.3.2)
     selenium-devtools (0.129.0)
       selenium-webdriver (~> 4.2)
-    selenium-webdriver (4.24.0)
+    selenium-webdriver (4.25.0)
       base64 (~> 0.2)
       logger (~> 1.4)
       rexml (~> 3.2, >= 3.2.5)
@@ -155,12 +155,12 @@ GEM
     xpath (3.2.0)
       nokogiri (~> 1.8)
     zeitwerk (2.6.8)
-    zorki (0.2.6)
+    zorki (0.2.8)
       apparition
       capybara
       oj
       selenium-devtools
-      selenium-webdriver (~> 4.24.0)
+      selenium-webdriver (~> 4.25.0)
       typhoeus
 
 PLATFORMS
@@ -172,7 +172,7 @@ DEPENDENCIES
   curb (~> 1.0, >= 1.0.5)
   dotenv (~> 2.7.6)
   forki!
-  minitest (~> 5.0)
+  minitest (~> 5.25)
   rack (= 2.2.4)
   rake (~> 13.0)
   rubocop (~> 1.7)

data/lib/forki/scrapers/post_scraper.rb

@@ -23,6 +23,7 @@ module Forki
       raise ContentUnavailableError unless is_post_available?
 
       graphql_objects = get_graphql_objects(graphql_strings)
+      post_is_text_only = check_if_post_is_text_only(graphql_objects)
       post_has_video = check_if_post_is_video(graphql_objects)
       post_has_image = check_if_post_is_image(graphql_objects)
 
@@ -30,7 +31,9 @@ module Forki
       # https://www.facebook.com/PlandemicMovie/posts/588866298398729/
       post_has_video_in_comment_stream = check_if_post_is_in_comment_stream(graphql_objects) if post_has_video == false
 
-      if post_has_video
+      if post_is_text_only
+        extract_text_post_data(graphql_objects)
+      elsif post_has_video
         extract_video_post_data(graphql_strings)
       elsif post_has_video_in_comment_stream
         extract_video_comment_post_data(graphql_objects)
@@ -45,6 +48,21 @@ module Forki
       graphql_strings.map { |graphql_object| JSON.parse(graphql_object) }
     end
 
+    def check_if_post_is_text_only(graphql_objects)
+      graphql_object = graphql_objects.find do |graphql_object|
+        # next unless graphql_object.key?("nodes")
+        next if graphql_object.dig("node", "comet_sections", "content", "story", "comet_sections", "message", "story", "is_text_only_story").nil?
+        # next unless graphql_object.to_s.include?("is_text_only_story")
+        # graphql_nodes = graphql_object["nodes"]
+        graphql_object.dig("node", "comet_sections", "content", "story", "comet_sections", "message", "story", "is_text_only_story")
+      end
+
+      return false if graphql_object.nil?
+
+      return true if graphql_object.dig("node", "comet_sections", "content", "story", "comet_sections", "message", "story", "is_text_only_story")
+      false
+    end
+
     def check_if_post_is_video(graphql_objects)
       graphql_objects.any? { |graphql_object| graphql_object.key?("is_live_streaming") || graphql_object.key?("video") || check_if_post_is_reel(graphql_object) }
     end
@@ -124,6 +142,67 @@ module Forki
       false
     end
 
+    def extract_text_post_data(graphql_objects)
+      graphql_object = graphql_objects.find do |graphql_object|
+        next if graphql_object.dig("node", "comet_sections", "content", "story", "comet_sections", "message", "story", "is_text_only_story").nil?
+        graphql_object
+      end
+
+      unless graphql_object.nil? || graphql_object.count.zero?
+        if graphql_object["node"]["comet_sections"]["feedback"]["story"].has_key?("story_ufi_container")
+          feedback_object = graphql_object["node"]["comet_sections"]["feedback"]["story"]["story_ufi_container"]["story"]["feedback_context"]["feedback_target_with_context"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
+        elsif graphql_object["node"]["comet_sections"]["feedback"]["story"].dig("feedback_context")
+          begin
+            feedback_object = graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
+          rescue NoMethodError; end
+        elsif graphql_object["node"]["comet_sections"]["feedback"]["story"].has_key?("comet_feed_ufi_container")
+          feedback_object = graphql_object["node"]["comet_sections"]["feedback"]["story"]["comet_feed_ufi_container"]["story"]["story_ufi_container"]["story"]["feedback_context"]["feedback_target_with_context"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
+        else
+          feedback_object = graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
+        end
+
+        if feedback_object.has_key?("cannot_see_top_custom_reactions")
+          reaction_counts = extract_reaction_counts(feedback_object["cannot_see_top_custom_reactions"]["top_reactions"])
+        else
+          reaction_counts = extract_reaction_counts(feedback_object["top_reactions"])
+        end
+
+        id = graphql_object["node"]["post_id"]
+        num_comments = feedback_object["comments_count_summary_renderer"]["feedback"]["comment_rendering_instance"]["comments"]["total_count"]
+        reshare_warning = feedback_object["should_show_reshare_warning"]
+        share_count_object = feedback_object.fetch("share_count", {})
+        num_shares = share_count_object.fetch("count", nil)
+
+        text = graphql_object["node"]["comet_sections"]["content"]["story"].dig("message", "text")
+        text = "" if text.nil?
+
+        profile_link = graphql_object["node"]["comet_sections"]["content"]["story"]["actors"].first["url"]
+
+        unless graphql_object["node"]["comet_sections"].dig("content", "story", "comet_sections", "context_layout", "story", "comet_sections", "metadata").nil?
+          created_at = graphql_object["node"]["comet_sections"].dig("content", "story", "comet_sections", "context_layout", "story", "comet_sections", "metadata")&.first["story"]["creation_time"]
+        else
+          created_at = graphql_object["node"]["comet_sections"]["context_layout"]["story"]["comet_sections"]["metadata"].first["story"]["creation_time"]
+        end
+
+        has_video = false
+      end
+
+      post_details = {
+        id: id,
+        num_comments: num_comments,
+        num_shares: num_shares,
+        reshare_warning: reshare_warning,
+        image_url: nil,
+        text: text,
+        profile_link: profile_link,
+        created_at: created_at,
+        has_video: has_video
+      }
+      post_details[:image_file] = []
+      post_details[:reactions] = reaction_counts
+      post_details
+    end
+
     def extract_video_comment_post_data(graphql_objects)
       graphql_nodes = nil
       graphql_objects.find do |graphql_object|
@@ -185,7 +264,7 @@ module Forki
         media_object = story_node_object["comet_sections"]["content"]["story"]["attachments"].first["styles"]["attachment"]
         if media_object.has_key?("video")
           video_object = story_node_object["comet_sections"]["content"]["story"]["attachments"].first["styles"]["attachment"]["media"]["video"]
-        elsif media_object.has_key?("media") && media_object["media"].has_key?("browser_native_sd_url")
+        elsif media_object.has_key?("media") && (media_object["media"].has_key?("browser_native_sd_url") || media_object["media"].has_key?("videoDeliveryLegacyFields"))
           video_object = media_object["media"]
         end
 
@@ -250,6 +329,12 @@ module Forki
         reshare_warning = feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"]
       end
 
+      if !video_object.key?("browser_native_hd_url") && !video_object.key?("browser_native_sd_url") && video_object.key?("videoDeliveryLegacyFields")
+        video_url = video_object["videoDeliveryLegacyFields"]["browser_native_hd_url"] || video_object["videoDeliveryLegacyFields"]["browser_native_sd_url"]
+      else
+        video_url = video_object["browser_native_hd_url"] || video_object["browser_native_sd_url"]
+      end
+
       post_details = {
         id: video_object["id"],
         num_comments: num_comments,
@@ -257,7 +342,7 @@ module Forki
         num_views: view_count,
         reshare_warning: reshare_warning,
         video_preview_image_url: video_object["preferred_thumbnail"]["image"]["uri"],
-        video_url: video_object["browser_native_hd_url"] || video_object["browser_native_sd_url"],
+        video_url: video_url,
         text: text,
         created_at: creation_date,
         profile_link: story_node_object["comet_sections"]["context_layout"]["story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]["url"],
@@ -272,6 +357,9 @@ module Forki
     def extract_video_post_data_alternative(graphql_object_array)
       sidepane_object = graphql_object_array.find { |graphql_object| graphql_object.key?("tahoe_sidepane_renderer") }
       video_object = graphql_object_array.find { |graphql_object| graphql_object.has_key?("video") }
+
+      raise Forki::ContentUnavailableError if sidepane_object.nil? && video_object.nil?
+
       feedback_object = sidepane_object["tahoe_sidepane_renderer"]["video"]["feedback"]
 
       if sidepane_object["tahoe_sidepane_renderer"]["video"]["feedback"].key?("cannot_see_top_custom_reactions")
@@ -288,9 +376,14 @@ module Forki
         num_comments = feedback_object["comments_count_summary_renderer"]["feedback"]["total_comment_count"]
       end
 
-      text = sidepane_object["tahoe_sidepane_renderer"]["video"]["creation_story"]["comet_sections"]["message"].dig(["story", "message", "text"])
+      text = sidepane_object["tahoe_sidepane_renderer"]["video"]["creation_story"]["comet_sections"].dig("message", "story", "message", "text")
       text = "" if text.nil?
 
+      video_url = video_object["video"]["playable_url_quality_hd"] || video_object["video"]["browser_native_hd_url"] || video_object["video"]["browser_native_sd_url"] || video_object["video"]["playable_url"]
+      if video_url.nil? && video_object["video"].key?("videoDeliveryLegacyFields")
+        video_url = video_object["video"]["videoDeliveryLegacyFields"]["browser_native_hd_url"] || video_object["video"]["videoDeliveryLegacyFields"]["browser_native_sd_url"]
+      end
+
       post_details = {
         id: video_object["id"],
         num_comments: num_comments,
@@ -298,7 +391,7 @@ module Forki
         num_views: feedback_object["video_view_count"],
         reshare_warning: feedback_object["should_show_reshare_warning"],
         video_preview_image_url: video_object["video"]["preferred_thumbnail"]["image"]["uri"],
-        video_url: video_object["video"]["playable_url_quality_hd"] || video_object["video"]["browser_native_hd_url"] || video_object["video"]["browser_native_sd_url"] || video_object["video"]["playable_url"],
+        video_url: video_url,
         text: text,
         created_at: video_object["video"]["publish_time"],
         profile_link: sidepane_object["tahoe_sidepane_renderer"]["video"]["creation_story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]["url"],
@@ -314,7 +407,6 @@ module Forki
     # Extracts data from an image post by parsing GraphQL strings as seen in the video post scraper above
     def extract_image_post_data(graphql_object_array)
       # This is a weird one-off style
-
       graphql_object = graphql_object_array.find { |graphql_object| !graphql_object.dig("node", "comet_sections", "content", "story", "attachments").nil? }
       unless graphql_object.nil? || graphql_object.count.zero?
         # TODO: These two branches are *super* similar, probably a lot of overlap
@@ -352,7 +444,7 @@ module Forki
           end
         end
 
-        text = graphql_object["node"]["comet_sections"]["content"]["story"].dig(["message", "text"])
+        text = graphql_object["node"]["comet_sections"]["content"]["story"].dig("message", "text")
         text = "" if text.nil?
 
         profile_link = graphql_object["node"]["comet_sections"]["content"]["story"]["actors"].first["url"]
@@ -367,6 +459,8 @@ module Forki
       else
         graphql_object_array.find { |graphql_object| graphql_object.key?("viewer_actor") && graphql_object.key?("display_comments") }
         curr_media_object = graphql_object_array.find { |graphql_object| graphql_object.key?("currMedia") }
+        raise Forki::ContentUnavailableError if curr_media_object.nil?
+
         creation_story_object = graphql_object_array.find { |graphql_object| graphql_object.key?("creation_story") && graphql_object.key?("message") }
 
         feedback_object = graphql_object_array.find { |graphql_object| graphql_object.has_key?("comet_ufi_summary_and_actions_renderer") }["comet_ufi_summary_and_actions_renderer"]["feedback"]

data/lib/forki/scrapers/sieves/video_sieves/video_sieve.rb

@@ -4,7 +4,6 @@ class VideoSieve
   end
 
   def self.sieve_for_graphql_objects(graphql_objects)
-
     sieve = sieve_class_for_graphql_objects(graphql_objects)
     return nil if sieve.nil?
 
@@ -19,6 +18,6 @@ private
   end
 end
 
-Dir[File.join(__dir__, '*.rb')].each do |file|
+Dir[File.join(__dir__, "*.rb")].each do |file|
   require file unless file.end_with?("video_sieve.rb")
 end

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_reel.rb

@@ -15,7 +15,7 @@ class VideoSieveReel < VideoSieve
 
     true
   rescue StandardError
-    return false
+    false
   end
 
   # output the expected format of:
@@ -56,7 +56,7 @@ class VideoSieveReel < VideoSieve
     video_preview_image_url = video_object["short_form_video_context"]["playback_video"]["preferred_thumbnail"]["image"]["uri"]
     video_url = video_object["short_form_video_context"]["playback_video"]["browser_native_hd_url"] || video_object["short_form_video_context"]["playback_video"]["browser_native_sd_url"]
 
-    post_details = {
+    {
       id: video_object["short_form_video_context"]["video"]["id"],
       num_comments: feedback_object["feedback"]["top_level_comments"]["totalCountIncludingReplies"],
       num_shared: Forki::Scraper.extract_int_from_num_element(feedback_object["feedback"]["share_count_reduced"]),
@@ -74,7 +74,7 @@ class VideoSieveReel < VideoSieve
     }
   end
 
-  private
+private
 
   def self.extractor(graphql_objects)
     video_objects = graphql_objects.filter do |go|

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_reel_2.rb

@@ -14,7 +14,7 @@ class VideoSieveReel2 < VideoSieve
 
     true
   rescue StandardError
-    return false
+    false
   end
 
   # output the expected format of:
@@ -39,7 +39,6 @@ class VideoSieveReel2 < VideoSieve
   def self.sieve(graphql_objects)
     video_object = self.extractor(graphql_objects)
 
-
     feedback_object = graphql_objects.filter do |go|
       go = go.first if go.kind_of?(Array) && !go.empty?
       !go.dig("feedback", "total_comment_count").nil?
@@ -55,7 +54,12 @@ class VideoSieveReel2 < VideoSieve
     video_preview_image_url = video_object["short_form_video_context"]["playback_video"]["preferred_thumbnail"]["image"]["uri"]
     video_url = video_object["short_form_video_context"]["playback_video"]["browser_native_hd_url"] || video_object["short_form_video_context"]["playback_video"]["browser_native_sd_url"]
 
-    post_details = {
+    if video_url.nil? && video_object["short_form_video_context"]["playback_video"].has_key?("videoDeliveryLegacyFields")
+      video_url = video_object["short_form_video_context"]["playback_video"]["videoDeliveryLegacyFields"]["browser_native_hd_url"]
+      video_url = video_object["short_form_video_context"]["playback_video"]["videoDeliveryLegacyFields"]["browser_native_sd_url"] if video_url.nil?
+    end
+
+    {
       id: video_object["short_form_video_context"]["video"]["id"],
       num_comments: feedback_object["feedback"]["total_comment_count"],
       num_shared: Forki::Scraper.extract_int_from_num_element(feedback_object["feedback"]["share_count_reduced"]),
@@ -71,11 +75,9 @@ class VideoSieveReel2 < VideoSieve
       video_file: Forki.retrieve_media(video_url),
       reactions: nil # Only available on comments it seems? Look into this again sometime
     }
-  rescue StandardError => e
-    debugger
   end
 
-  private
+private
 
   def self.extractor(graphql_objects)
     video_objects = graphql_objects.filter do |go|

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_video_page.rb

@@ -9,7 +9,7 @@ class VideoSieveVideoPage < VideoSieve
     return false unless feedback_object.has_key?("cannot_see_top_custom_reactions")
 
     true
-  rescue StandardError => e
+  rescue StandardError
     false
   end
 
@@ -61,7 +61,7 @@ class VideoSieveVideoPage < VideoSieve
     }
   end
 
-  private
+private
 
   def self.extractor(graphql_objects)
     story_node_object = graphql_objects.find { |graphql_object| graphql_object.key? "node" }&.fetch("node", nil) # user posted video

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_video_page_2.rb

@@ -61,7 +61,7 @@ class VideoSieveVideoPage2 < VideoSieve
     }
   end
 
-  private
+private
 
   def self.extractor(graphql_objects)
     story_node_object = graphql_objects.find { |graphql_object| graphql_object.key? "node" }&.fetch("node", nil) # user posted video

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_watch_tab.rb

@@ -12,11 +12,11 @@ class VideoSieveWatchTab < VideoSieve
     return false unless video_object.kind_of?(Array) && !video_object.empty?
 
     video_object = video_object.first
-    return false unless video_object.kind_of?(Hash) && video_object.keys.include?("media")
+    return false unless video_object.kind_of?(Hash) && video_object.key?("media")
 
     true
   rescue StandardError
-    return false
+    false
   end
 
   # output the expected format of:
@@ -46,11 +46,16 @@ class VideoSieveWatchTab < VideoSieve
     video_url = video_object.dig("short_form_video_context", "playback_video", "browser_native_hd_url") if video_url.nil?
     video_url = video_object.dig("short_form_video_context", "playback_video", "browser_native_sd_url") if video_url.nil?
 
-    debugger if video_url.nil?
+    video_url = video_object["attachments"]&.first.dig("media", "videoDeliveryLegacyFields", "browser_native_hd_url") if video_url.nil?
+    video_url = video_object["attachments"]&.first.dig("media", "videoDeliveryLegacyFields", "browser_native_sd_url") if video_url.nil?
+
+    raise VideoSieveFailedError.new(sieve_class: "VideoSieveWatchTab") if video_url.nil?
 
     video_preview_image_url = video_object["attachments"]&.first.dig("media", "preferred_thumbnail", "image", "uri")
     video_preview_image_url = video_object["short_form_video_context"]["video"]["first_frame_thumbnail"] if video_preview_image_url.nil?
 
+    raise VideoSieveFailedError.new(sieve_class: "VideoSieveWatchTab") if video_preview_image_url.nil?
+
     if !video_object["feedback_context"].nil?
       feedback_object = video_object["feedback_context"]["feedback_target_with_context"]
     else
@@ -60,7 +65,7 @@ class VideoSieveWatchTab < VideoSieve
 
     begin
       profile_link = video_object["attachments"].first["media"]["owner"]["url"]
-    rescue StandardError => e
+    rescue StandardError
       profile_link = video_object["short_form_video_context"]["video_owner"]["url"]
     end
 
@@ -75,11 +80,11 @@ class VideoSieveWatchTab < VideoSieve
       else
         reactions = feedback_object["top_reactions"]["edges"]
       end
-    rescue StandardError => e
+    rescue StandardError
       reactions = feedback_object["unified_reactors"]["count"]
     end
 
-    post_details = {
+    {
       id: video_object.dig("shareable", "id") || video_object["attachments"].first["media"]["id"],
       num_comments: feedback_object["total_comment_count"],
       num_shared: nil, # This is not associated with these videos in this format
@@ -97,7 +102,7 @@ class VideoSieveWatchTab < VideoSieve
     }
   end
 
-  private
+private
 
   def self.extractor(graphql_objects)
     video_objects = graphql_objects.filter do |go|

data/lib/forki/scrapers/user_scraper.rb

@@ -124,7 +124,7 @@ module Forki
       uri = URI(url)
       query = uri.query
       components = URI.decode_uri_component(query)
-      extracted_url = URI.extract(components).first
+      extracted_url = URI::Parser.new.extract(components).first
       extracted_uri = URI(extracted_url)
       username = extracted_uri.to_s.match(/(https:\/\/www.instagram.com\/_u\/[\w]+)/).to_s.split("/").last
 

data/lib/forki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Forki
-  VERSION = "0.2.14"
+  VERSION = "0.2.16"
 end

data/lib/forki.rb

@@ -48,6 +48,13 @@ module Forki
     end
   end
 
+  class VideoSieveFailedError < StandardError
+    def initialize(msg = "Video sieve failed to find a video", sieve_class: VideoSieve)
+      self.msg = "#{sieve_class} failed to find a video" if msg.nil?
+      super
+    end
+  end
+
   define_setting :temp_storage_location, "tmp/forki"
 
   # Extract the file extension from a media URL

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forki
 version: !ruby/object:Gem::Version
-  version: 0.2.14
+  version: 0.2.16
 platform: ruby
 authors:
 - ''
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-23 00:00:00.000000000 Z
+date: 2024-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara