forki 0.2.1 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 412d3450d71c6ec80cbf868cf005545ed437a3bb06532cb383af0fa65088baaf
-  data.tar.gz: 78707c68f6c49f65f755c4df5edc08585aacb05dd1f22311fca734285e3f74a4
+  metadata.gz: f1c8304ed1312aa0e5a7f2f8f5ad2de9a407c761fb7ab9212d66465f599caf6f
+  data.tar.gz: e35a13415b85b9bd57cd9028841c61a324e4011e1a449cd102fe470015ed670d
 SHA512:
-  metadata.gz: bde0f077222e99c19da6ac3a9b1dad259d21fa147fc18d6ba52a3e9706aa4ef4408379b59a5c95e2e6b9d5377b1b8272b78ba934f903caa45db94a6da031c803
-  data.tar.gz: 325224a93a9183695c54024f0de22d2882aa04998fd6d31e2db4f6226c217ebd44de4fd140f3ebdd06bb5edb2ff01b2b0d74f70c2a804915461091fc81482d68
+  metadata.gz: 4179d91435cdc9c82c39c92a15f3df4e7e437f250eccf01e79cf9cb631c326b76badaed3e5851cc26c91713c0cdc27feba56127e8d8c647525960286efd50412
+  data.tar.gz: b5f8ca4461440a785d544017fc1dbc72e9edfdc1d6924717813cf7cd1bfb67fa8695ecc072d00fd476d55eee3ecedca710c107a11bbfe89e8d4b067f443040ae

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    forki (0.1.4)
+    forki (0.2.1)
       apparition
       capybara
       oj
@@ -127,7 +127,7 @@ GEM
       rubocop-rails (~> 2.0)
     ruby-progressbar (1.13.0)
     rubyzip (2.3.2)
-    selenium-webdriver (4.10.0)
+    selenium-webdriver (4.11.0)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)

data/bin/generator_templates/image_sieve_test_template.rb.erb

@@ -29,7 +29,7 @@ class ImageSieve<%= camel_name %>Test < Minitest::Test
     assert_equal false, result[:reshare_warning]
     assert_not_nil result[:video_preview_image_url]
     assert_not_nil result[:video_url]
-    assert_equal nil, text
+    assert_equal nil, result[:text]
     assert_equal 1654989063, result[:created_at]
     assert_equal nil, result[:profile_link]
     assert_equal false, result[:has_video]

data/bin/generator_templates/video_sieve_test_template.rb.erb

@@ -28,7 +28,7 @@ class VideoSieve<%= camel_name %>Test < Minitest::Test
     assert_equal false, result[:reshare_warning]
     assert_not_nil result[:video_preview_image_url]
     assert_not_nil result[:video_url]
-    assert_equal nil, text
+    assert_equal nil, result[:text]
     assert_equal 1654989063, result[:created_at]
     assert_equal nil, result[:profile_link]
     assert_equal true, result[:has_video]

data/lib/forki/scrapers/post_scraper.rb

@@ -15,7 +15,7 @@ module Forki
       views_pattern = /[0-9MK, ]+Views/
       spans = all("span")
       views_span = spans.find { |s| s.text(:all) =~ views_pattern }
-      extract_int_from_num_element(views_span)
+      Scraper.extract_int_from_num_element(views_span)
     end
 
     def extract_post_data(graphql_strings)
@@ -215,7 +215,7 @@ module Forki
 
     def extract_video_post_data_alternative(graphql_object_array)
       sidepane_object = graphql_object_array.find { |graphql_object| graphql_object.key?("tahoe_sidepane_renderer") }
-      video_object = graphql_object_array.find { |graphql_object| graphql_object.keys == ["video"] }
+      video_object = graphql_object_array.find { |graphql_object| graphql_object.has_key?("video") }
       feedback_object = sidepane_object["tahoe_sidepane_renderer"]["video"]["feedback"]
       reaction_counts = extract_reaction_counts(sidepane_object["tahoe_sidepane_renderer"]["video"]["feedback"]["cannot_see_top_custom_reactions"]["top_reactions"])
       share_count_object = feedback_object.fetch("share_count", {})

data/lib/forki/scrapers/scraper.rb

@@ -209,7 +209,7 @@ module Forki
     # Extracts an integer out of a string describing a number
     # e.g. "4K Comments" returns 4000
     # e.g. "131 Shares" returns 131
-    def extract_int_from_num_element(element)
+    def self.extract_int_from_num_element(element)
       return unless element
 
       if element.class != String # if an html element was passed in

data/lib/forki/scrapers/sieves/video_sieves/video_sieve.rb

@@ -14,7 +14,7 @@ class VideoSieve
 private
 
   def self.sieve_class_for_graphql_objects(graphql_objects)
-    sieves = [VideoSieveWatchTab, VideoSieveVideoPage]
+    sieves = [VideoSieveWatchTab, VideoSieveVideoPage, VideoSieveReel]
     sieves.detect { |sieve| sieve.check(graphql_objects) }
   end
 end

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_reel.rb

@@ -0,0 +1,78 @@
+class VideoSieveReel < VideoSieve
+  # To check if it's valid for the inputted graphql objects
+  def self.check(graphql_objects)
+    video_object = self.extractor(graphql_objects)
+
+    return false unless video_object.has_key?("short_form_video_context")
+    true
+  rescue StandardError
+    return false
+  end
+
+  # output the expected format of:
+  #
+  # post_details = {
+  #   id: video_object["id"],
+  #   num_comments: num_comments,
+  #   num_shares: share_count_object.fetch("count", nil),
+  #   num_views: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["video_view_count"],
+  #   reshare_warning: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"],
+  #   video_preview_image_url: video_object["preferred_thumbnail"]["image"]["uri"],
+  #   video_url: video_object["browser_native_hd_url"] || video_object["browser_native_sd_url"],
+  #   text: text,
+  #   created_at: creation_date,
+  #   profile_link: story_node_object["comet_sections"]["context_layout"]["story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]["url"],
+  #   has_video: true
+  # }
+  # post_details[:video_preview_image_file] = Forki.retrieve_media(post_details[:video_preview_image_url])
+  # post_details[:video_file] = Forki.retrieve_media(post_details[:video_url])
+  # post_details[:reactions] = reaction_counts
+
+  def self.sieve(graphql_objects)
+    video_object = self.extractor(graphql_objects)
+
+
+    feedback_object = graphql_objects.filter do |go|
+      go = go.first if go.kind_of?(Array) && !go.empty?
+      !go.dig("feedback", "top_level_comments").nil?
+    end.first
+
+    reels_feedback_renderer = graphql_objects.filter do |go|
+      go.dig("reels_feedback_renderer")
+    end.first
+
+    reels_feedback_renderer["reels_feedback_renderer"]["story"]
+    reshare_warning = video_object["short_form_video_context"]["playback_video"].dig("warning_screen_renderer", "cix_screen", "view_model", "__typename") == "OverlayWarningScreenViewModel"
+
+    video_preview_image_url = video_object["short_form_video_context"]["playback_video"]["preferred_thumbnail"]["image"]["uri"]
+    video_url = video_object["short_form_video_context"]["playback_video"]["browser_native_hd_url"] || video_object["short_form_video_context"]["playback_video"]["browser_native_sd_url"]
+
+    post_details = {
+      id: video_object["short_form_video_context"]["video"]["id"],
+      num_comments: feedback_object["feedback"]["top_level_comments"]["totalCountIncludingReplies"],
+      num_shared: Forki::Scraper.extract_int_from_num_element(feedback_object["feedback"]["share_count_reduced"]),
+      num_views: nil,
+      reshare_warning: reshare_warning,
+      video_preview_image_url: video_preview_image_url,
+      video_url: video_url,
+      text: nil, # Reels don't have text
+      created_at: JSON.parse(feedback_object["tracking"])["page_insights"].first[1]["post_context"]["publish_time"], # Yea, this is weird
+      profile_link: video_object["short_form_video_context"]["video_owner"]["url"],
+      has_video: true,
+      video_preview_image_file: Forki.retrieve_media(video_preview_image_url),
+      video_file: Forki.retrieve_media(video_url),
+      reactions: nil # Only available on comments it seems? Look into this again sometime
+    }
+  end
+
+  private
+
+  def self.extractor(graphql_objects)
+    video_objects = graphql_objects.filter do |go|
+      go = go.first if go.kind_of?(Array) && !go.empty?
+      go.has_key?("video")
+    end
+
+    video_objects.first.dig("video", "creation_story")
+  end
+end

data/lib/forki/scrapers/user_scraper.rb

@@ -10,7 +10,7 @@ module Forki
 
       return nil if number_of_likes_match.nil?
 
-      extract_int_from_num_element(number_of_likes_match.named_captures["num_likes"])
+      Scraper.extract_int_from_num_element(number_of_likes_match.named_captures["num_likes"])
     end
 
     # Finds and returns the number of people who follow the current page
@@ -21,12 +21,12 @@ module Forki
 
       return nil if number_of_followers_match.nil?
 
-      number_of_followers = extract_int_from_num_element(number_of_followers_match.named_captures["num_followers"])
+      number_of_followers = Scraper.extract_int_from_num_element(number_of_followers_match.named_captures["num_followers"])
 
       # Note, this is sticking around if we want to use it later
       # if number_of_followers.nil?
       #   number_of_followers_string = JSON.parse(profile_header_str)["user"]["profile_header_renderer"]["user"]["profile_social_context"]["content"].first["text"]["text"]
-      #   number_of_followers = extract_int_from_num_element(number_of_followers_string)
+      #   number_of_followers = Scraper.extract_int_from_num_element(number_of_followers_string)
       # end
 
       number_of_followers
@@ -36,7 +36,7 @@ module Forki
       followers_string = profile_followers_node["node"]["timeline_context_item"]["renderer"]["context_item"]["title"]["text"]
       followers_pattern = /[0-9,]+/
       number_of_followers_match = followers_pattern.match(followers_string).to_s
-      extract_int_from_num_element(number_of_followers_match)
+      Scraper.extract_int_from_num_element(number_of_followers_match)
     end
 
     # Returns a hash of details about a Facebook user profile

data/lib/forki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Forki
-  VERSION = "0.2.1"
+  VERSION = "0.2.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forki
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.3
 platform: ruby
 authors:
 - ''
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-08-01 00:00:00.000000000 Z
+date: 2023-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -126,6 +126,7 @@ files:
 - lib/forki/scrapers/scraper.rb
 - lib/forki/scrapers/sieves/image_sieves/image_sieve.rb.rb
 - lib/forki/scrapers/sieves/video_sieves/video_sieve.rb
+- lib/forki/scrapers/sieves/video_sieves/video_sieve_reel.rb
 - lib/forki/scrapers/sieves/video_sieves/video_sieve_video_page.rb
 - lib/forki/scrapers/sieves/video_sieves/video_sieve_watch_tab.rb
 - lib/forki/scrapers/user_scraper.rb