forki 0.2.7 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf93e965787eaf05b26f6ea1377775fb61ed131a52458a371336474d09e4a639
-  data.tar.gz: 729e9409bf76eb8551913f64e02d3905a878d057f045e16a64f712926d0d5cc8
+  metadata.gz: be729f6e73f3a233ef2337076e907a5f62abd6d82541d38cf59ac4cf97ba99ae
+  data.tar.gz: 8cd062ac0cf5688f60fd1d2afd6718013f7bd1a5a037194ee01135f9bb890f15
 SHA512:
-  metadata.gz: 6d710aee0bb1ae64c3796de31f85a9e39f26791bd12a653785544099d9e10629a3a239f69dff5702b80b009d1e3b6c9abfef50109e7078dcb70194f9f5c65384
-  data.tar.gz: 0b34b2dceaeff07c844e9fc8b42f27bfe3fbaae2c631bfe80f605f1c7caa821f0387780b37cc1bdb9b6735c20a967ba51e2d6a58a0a0478ee624520ab060402d
+  metadata.gz: f0f6002e321608fc472775ddeef2b476336ad2f104d7a82063c9b8d05ec15f6e795d09dfe67a9990653fb7297c9f82dc200032285143ef43a178b458801669ad
+  data.tar.gz: a8e5a9311783f97d6acbb628025ee6d1513ebdffe1e957b80658a3bf03f8261109fc50d4c7d765b4557046fd03288ce39627cc84deab774b6ac68023aa782dc3

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    forki (0.2.5)
+    forki (0.2.8)
       apparition
       capybara
       oj
@@ -35,14 +35,15 @@ GEM
       capybara (~> 3.13, < 4)
       websocket-driver (>= 0.6.5)
     ast (2.4.2)
-    bigdecimal (3.1.5)
+    base64 (0.2.0)
+    bigdecimal (3.1.8)
     builder (3.2.4)
     byebug (11.1.3)
-    capybara (3.39.2)
+    capybara (3.40.0)
       addressable
       matrix
       mini_mime (>= 0.1.3)
-      nokogiri (~> 1.8)
+      nokogiri (~> 1.11)
       rack (>= 1.6.0)
       rack-test (>= 0.6.3)
       regexp_parser (>= 1.5, < 3.0)
@@ -54,7 +55,7 @@ GEM
     erubi (1.12.0)
     ethon (0.16.0)
       ffi (>= 1.15.0)
-    ffi (1.16.3)
+    ffi (1.17.0-arm64-darwin)
     i18n (1.13.0)
       concurrent-ruby (~> 1.0)
     json (2.6.3)
@@ -67,12 +68,12 @@ GEM
     minitest (5.18.0)
     nokogiri (1.15.1-arm64-darwin)
       racc (~> 1.4)
-    oj (3.16.3)
+    oj (3.16.4)
       bigdecimal (>= 3.0)
     parallel (1.23.0)
     parser (3.2.2.1)
       ast (~> 2.4.1)
-    public_suffix (5.0.4)
+    public_suffix (5.1.1)
     racc (1.6.2)
     rack (2.2.4)
     rack-test (2.1.0)
@@ -129,7 +130,8 @@ GEM
       rubocop-rails (~> 2.0)
     ruby-progressbar (1.13.0)
     rubyzip (2.3.2)
-    selenium-webdriver (4.16.0)
+    selenium-webdriver (4.21.1)
+      base64 (~> 0.2)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)

data/lib/forki/scrapers/post_scraper.rb

@@ -37,7 +37,9 @@ module Forki
       elsif post_has_image
         extract_image_post_data(graphql_objects)
       else
-        raise UnhandledContentError
+        extract_image_post_data(graphql_objects)
+
+        #raise UnhandledContentError
       end
     end
 
@@ -202,7 +204,11 @@ module Forki
       begin
         feedback_object = story_node_object["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]
       rescue NoMethodError
-        feedback_object = story_node_object["comet_sections"]["feedback"]["story"]["comet_feed_ufi_container"]["story"]["story_ufi_container"]["story"]["feedback_context"]["feedback_target_with_context"]
+        begin
+          feedback_object = story_node_object["comet_sections"]["feedback"]["story"]["comet_feed_ufi_container"]["story"]["story_ufi_container"]["story"]["feedback_context"]["feedback_target_with_context"]
+        rescue NoMethodError
+          feedback_object = story_node_object["comet_sections"]["feedback"]["story"]["story_ufi_container"]["story"]["feedback_context"]["feedback_target_with_context"]
+        end
       end
 
       if feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"].key?("cannot_see_top_custom_reactions")
@@ -313,8 +319,14 @@ module Forki
         # TODO: These two branches are *super* similar, probably a lot of overlap
         attachments = graphql_object["node"]["comet_sections"]["content"]["story"]["attachments"]
 
-        if graphql_object["node"]["comet_sections"]["feedback"]["story"].key?("feedback_context")
-          feedback_object = graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
+        if graphql_object["node"]["comet_sections"]["feedback"]["story"].has_key?("story_ufi_container")
+          feedback_object = graphql_object["node"]["comet_sections"]["feedback"]["story"]["story_ufi_container"]["story"]["feedback_context"]["feedback_target_with_context"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
+        elsif graphql_object["node"]["comet_sections"]["feedback"]["story"].dig("feedback_context")
+          begin
+            feedback_object = graphql_object["node"]["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
+          rescue NoMethodError
+            debugger
+          end
         elsif graphql_object["node"]["comet_sections"]["feedback"]["story"].has_key?("comet_feed_ufi_container")
           feedback_object = graphql_object["node"]["comet_sections"]["feedback"]["story"]["comet_feed_ufi_container"]["story"]["story_ufi_container"]["story"]["feedback_context"]["feedback_target_with_context"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
         else
@@ -331,13 +343,13 @@ module Forki
         num_comments = feedback_object["comments_count_summary_renderer"]["feedback"]["comment_rendering_instance"]["comments"]["total_count"]
         reshare_warning = feedback_object["should_show_reshare_warning"]
 
-        if attachments.first["styles"]["attachment"].key?("all_subattachments")
+        if attachments.count.positive? && attachments.first["styles"]["attachment"]&.key?("all_subattachments")
           image_url = attachments.first["styles"]["attachment"]["all_subattachments"]["nodes"].first["media"]["image"]["uri"]
         else
-          image_url = attachments.first.dig("styles", "attachment", "media", "photo_image", "uri")
+          image_url = attachments.first&.dig("styles", "attachment", "media", "photo_image", "uri")
 
           if image_url.nil?
-            image_url = attachments.first["styles"]["attachment"]["media"]["large_share_image"]["uri"]
+            image_url = attachments.first&.dig("styles", "attachment", "media", "large_share_image", "uri")
           end
         end
 
@@ -502,6 +514,7 @@ module Forki
       graphql_strings = find_graphql_data_strings(page.html)
 
       post_data = extract_post_data(graphql_strings)
+
       post_data[:url] = url
       user_url = post_data[:profile_link]
 

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_watch_tab.rb

@@ -41,8 +41,12 @@ class VideoSieveWatchTab < VideoSieve
   def self.sieve(graphql_objects)
     video_object = self.extractor(graphql_objects)
 
-    video_url = video_object["attachments"].first["media"]["browser_native_sd_url"]
-    video_preview_image_url = video_object["attachments"].first["media"]["preferred_thumbnail"]["image"]["uri"]
+    # video_url = video_object["attachments"].first["media"]["browser_native_sd_url"]
+    video_url = video_object["short_form_video_context"]["playback_video"]["browser_native_hd_url"]
+    video_url = video_object["short_form_video_context"]["playback_video"]["browser_native_sd_url"] if video_url.nil?
+
+    # video_preview_image_url = video_object["attachments"].first["media"]["preferred_thumbnail"]["image"]["uri"]
+    video_preview_image_url = video_object["short_form_video_context"]["video"]["first_frame_thumbnail"]
 
     if !video_object["feedback_context"].nil?
       feedback_object = video_object["feedback_context"]["feedback_target_with_context"]
@@ -51,16 +55,25 @@ class VideoSieveWatchTab < VideoSieve
       feedback_object = feedback_object["feedback"] if feedback_object.has_key?("feedback")
     end
 
-    profile_link = video_object["attachments"].first["media"]["owner"]["url"]
+    begin
+      profile_link = video_object["attachments"].first["media"]["owner"]["url"]
+    rescue StandardError => e
+      profile_link = video_object["short_form_video_context"]["video_owner"]["url"]
+    end
+
     if profile_link.nil?
       filtered_json = graphql_objects.find { |go| go.has_key? "attachments" }
       profile_link = filtered_json["attachments"].first["media"]["creation_story"]["comet_sections"]["title"]["story"]["actors"].first["url"]
     end
 
-    if feedback_object.key?("cannot_see_top_custom_reactions")
-      reactions = feedback_object["cannot_see_top_custom_reactions"]["top_reactions"]["edges"]
-    else
-      reactions = feedback_object["top_reactions"]["edges"]
+    begin
+      if feedback_object.key?("cannot_see_top_custom_reactions")
+        reactions = feedback_object["cannot_see_top_custom_reactions"]["top_reactions"]["edges"]
+      else
+        reactions = feedback_object["top_reactions"]["edges"]
+      end
+    rescue StandardError => e
+      reactions = feedback_object["unified_reactors"]["count"]
     end
 
     post_details = {

data/lib/forki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Forki
-  VERSION = "0.2.7"
+  VERSION = "0.2.9"
 end

data/lib/forki.rb

@@ -53,6 +53,8 @@ module Forki
   # Extract the file extension from a media URL
   # E.g. ".png" from https://scontent-atl3-2.xx.fbcdn.net/v/t39.30808-1.png?stp=dst-png_p148x148
   def self.extract_file_extension_from_url(url)
+    return nil if url.nil?
+
     stripped_url = url.split("?").first  # remove URL query params
     extension = stripped_url.split(".").last
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forki
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.2.9
 platform: ruby
 authors:
 - ''
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-05-08 00:00:00.000000000 Z
+date: 2024-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara