forki 0.2.2 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7edf04a9e64d21fa773c11ff10e0af20237061015195baac334e7de90750e7d8
-  data.tar.gz: 0bfc1308c2e54cc36c7454f8dbdc9b1776493350fdabfc1e166f2d9214a7da71
+  metadata.gz: f4868fd61b98809521249982e057f69d69018e8defba25792382f44e06b1921a
+  data.tar.gz: 1c700275aa5fafc19e4f6b42845e5d52c02e2bc3e6cd3f1082e1a59c5c4f472d
 SHA512:
-  metadata.gz: 90582f953c5490a2d2f338ecd3463dca1ebb4cad726307202aab90a71019a16476b42c46704ce0df462d71062117e2ca27dbd95ad2db06627d3be984e5528bbe
-  data.tar.gz: 671538d2aaeefa63fb4380252e3d167da7b65f05e1af3899f91de9ce1dd1ef53100d04dd72b868baf690e8b2187f55d2a092474a6140e3ddb35c17fe52766f66
+  metadata.gz: b5967db4c8e10b9d626767f041b2f59627170a49ceb4dc47e6da11f439a668bc656550e9b0c6a3a6a52180c8e75637ab2cbfb8a3c1b007bb91c4a5280e21aabd
+  data.tar.gz: 8230ec7b913fca196e255820066efab1c520e42ac38818652cb252966a16b8ce915f5f1224606cd5eddb81968fcdbd9f37506e028c641bcad2a402536174d321

data/lib/forki/scrapers/post_scraper.rb

@@ -393,7 +393,7 @@ module Forki
         sleep(5)
       end
 
-      # page.quit # Close browser between page navigations to prevent cache folder access issues
+      # page.quit # Close browser between page navigation to prevent cache folder access issues
 
       post_data[:user] = User.lookup(user_url).first
       page.quit
@@ -405,7 +405,12 @@ module Forki
     rescue StandardError => e
       raise e
     ensure
-      page.quit
+      # `page` here can be broken already. In which case we want to raise an error so it's retried later
+      begin
+        page.quit
+      rescue Curl::Err::ConnectionFailedError
+        raise Forki::RretryableError # This insures it'll eventually be retried by Hypatia
+      end
     end
   end
 end

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_reel.rb

@@ -0,0 +1,78 @@
+class VideoSieveReel < VideoSieve
+  # To check if it's valid for the inputted graphql objects
+  def self.check(graphql_objects)
+    video_object = self.extractor(graphql_objects)
+
+    return false unless video_object.has_key?("short_form_video_context")
+    true
+  rescue StandardError
+    return false
+  end
+
+  # output the expected format of:
+  #
+  # post_details = {
+  #   id: video_object["id"],
+  #   num_comments: num_comments,
+  #   num_shares: share_count_object.fetch("count", nil),
+  #   num_views: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["video_view_count"],
+  #   reshare_warning: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"],
+  #   video_preview_image_url: video_object["preferred_thumbnail"]["image"]["uri"],
+  #   video_url: video_object["browser_native_hd_url"] || video_object["browser_native_sd_url"],
+  #   text: text,
+  #   created_at: creation_date,
+  #   profile_link: story_node_object["comet_sections"]["context_layout"]["story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]["url"],
+  #   has_video: true
+  # }
+  # post_details[:video_preview_image_file] = Forki.retrieve_media(post_details[:video_preview_image_url])
+  # post_details[:video_file] = Forki.retrieve_media(post_details[:video_url])
+  # post_details[:reactions] = reaction_counts
+
+  def self.sieve(graphql_objects)
+    video_object = self.extractor(graphql_objects)
+
+
+    feedback_object = graphql_objects.filter do |go|
+      go = go.first if go.kind_of?(Array) && !go.empty?
+      !go.dig("feedback", "top_level_comments").nil?
+    end.first
+
+    reels_feedback_renderer = graphql_objects.filter do |go|
+      go.dig("reels_feedback_renderer")
+    end.first
+
+    reels_feedback_renderer["reels_feedback_renderer"]["story"]
+    reshare_warning = video_object["short_form_video_context"]["playback_video"].dig("warning_screen_renderer", "cix_screen", "view_model", "__typename") == "OverlayWarningScreenViewModel"
+
+    video_preview_image_url = video_object["short_form_video_context"]["playback_video"]["preferred_thumbnail"]["image"]["uri"]
+    video_url = video_object["short_form_video_context"]["playback_video"]["browser_native_hd_url"] || video_object["short_form_video_context"]["playback_video"]["browser_native_sd_url"]
+
+    post_details = {
+      id: video_object["short_form_video_context"]["video"]["id"],
+      num_comments: feedback_object["feedback"]["top_level_comments"]["totalCountIncludingReplies"],
+      num_shared: Forki::Scraper.extract_int_from_num_element(feedback_object["feedback"]["share_count_reduced"]),
+      num_views: nil,
+      reshare_warning: reshare_warning,
+      video_preview_image_url: video_preview_image_url,
+      video_url: video_url,
+      text: nil, # Reels don't have text
+      created_at: JSON.parse(feedback_object["tracking"])["page_insights"].first[1]["post_context"]["publish_time"], # Yea, this is weird
+      profile_link: video_object["short_form_video_context"]["video_owner"]["url"],
+      has_video: true,
+      video_preview_image_file: Forki.retrieve_media(video_preview_image_url),
+      video_file: Forki.retrieve_media(video_url),
+      reactions: nil # Only available on comments it seems? Look into this again sometime
+    }
+  end
+
+  private
+
+  def self.extractor(graphql_objects)
+    video_objects = graphql_objects.filter do |go|
+      go = go.first if go.kind_of?(Array) && !go.empty?
+      go.has_key?("video")
+    end
+
+    video_objects.first.dig("video", "creation_story")
+  end
+end

data/lib/forki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Forki
-  VERSION = "0.2.2"
+  VERSION = "0.2.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forki
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.4
 platform: ruby
 authors:
 - ''
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-08-03 00:00:00.000000000 Z
+date: 2023-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -126,6 +126,7 @@ files:
 - lib/forki/scrapers/scraper.rb
 - lib/forki/scrapers/sieves/image_sieves/image_sieve.rb.rb
 - lib/forki/scrapers/sieves/video_sieves/video_sieve.rb
+- lib/forki/scrapers/sieves/video_sieves/video_sieve_reel.rb
 - lib/forki/scrapers/sieves/video_sieves/video_sieve_video_page.rb
 - lib/forki/scrapers/sieves/video_sieves/video_sieve_watch_tab.rb
 - lib/forki/scrapers/user_scraper.rb
@@ -162,7 +163,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.14
+rubygems_version: 3.4.20
 signing_key:
 specification_version: 4
 summary: A gem to scrape Facebook pages for archive purposes.