forki 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a0667614af238aeb8089c1af60794918c0986b7f5e12fa6fe96d33a2a5c1a06f
-  data.tar.gz: 5668a5a6056bdf9bdd9c9fd8f119fa5ed754c5ee0b152d2764b26e9f9d7a8804
+  metadata.gz: 7a1b2f6a831ebac1bf9e79cc33818aa3d8f638459f538ff4f0cb92bab55b16df
+  data.tar.gz: 1df3b090db0ba37ecfbcf17933d41334620e9542a75f3328d9a335adec5d1ae9
 SHA512:
-  metadata.gz: 509ffb30dada2666236ed0900e1def8a61413ee3ba2e8705e31bd01422f8d2baf2cd24479faf906b6374602fcb0c1dc7bb334b7e02a560c798eba6efab30a2c9
-  data.tar.gz: 3f79b804b5505222e06a49352807d92c95f13a8735bedcd1b32cbf079be72b0922052014ef82da8771db108f7b0b674cd132caf7d377084c57cf0d245eb049e1
+  metadata.gz: b65c46157a6d1f320345d0216d138239c9d73cb29fd7fc0fa504655e0c789021a558d2e75a0447f026dc694e4fb95765cf12f8b2350e147b1011e670cd40621f
+  data.tar.gz: bc62a4decd0205e75ac93038d5768ff4d9ec55d8a25980dbb8522b3b2296f84e50db7caa3ba8aeb28d61eb482957b4d4bf68a9d9658cc26f4209198a64b1fe04

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    forki (0.1.1)
+    forki (0.1.4)
       apparition
       capybara
       oj
@@ -37,7 +37,7 @@ GEM
     ast (2.4.2)
     builder (3.2.4)
     byebug (11.1.3)
-    capybara (3.39.1)
+    capybara (3.39.2)
       addressable
       matrix
       mini_mime (>= 0.1.3)
@@ -66,11 +66,11 @@ GEM
     minitest (5.18.0)
     nokogiri (1.15.1-arm64-darwin)
       racc (~> 1.4)
-    oj (3.14.3)
+    oj (3.15.1)
     parallel (1.23.0)
     parser (3.2.2.1)
       ast (~> 2.4.1)
-    public_suffix (5.0.1)
+    public_suffix (5.0.3)
     racc (1.6.2)
     rack (2.2.4)
     rack-test (2.1.0)
@@ -127,7 +127,7 @@ GEM
       rubocop-rails (~> 2.0)
     ruby-progressbar (1.13.0)
     rubyzip (2.3.2)
-    selenium-webdriver (4.9.1)
+    selenium-webdriver (4.10.0)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
@@ -138,7 +138,7 @@ GEM
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.4.2)
     websocket (1.2.9)
-    websocket-driver (0.7.5)
+    websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     xpath (3.2.0)
@@ -160,6 +160,7 @@ DEPENDENCIES
   rubocop (~> 1.7)
   rubocop-rails (~> 2.17.3)
   rubocop-rails_config
+  thor
 
 BUNDLED WITH
    2.3.11

data/bin/generate_sieve

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "forki"
+require "thor"
+require "erb"
+
+class GenerateSieve < Thor
+
+  desc "generate TYPE NAME", "generate a TYPE named NAME"
+  option :video
+  option :image
+  def generate(type, name)
+    case type
+    when "post"
+      if !options[:video].nil?
+        style = :video
+      elsif !options[:image].nil?
+        style = :image
+      else
+        puts "Must indicate either video or image flag"
+        exit
+      end
+
+      SieveGenerator.generate_post_sieve(name, style)
+    when "user"
+      SieveGenerator.generate_user_sieve(name)
+    else
+      puts "Type must be `post` or `user` only. `#{type}` passed in."
+      exit
+    end
+  end
+end
+
+class SieveGenerator
+  def self.generate_post_sieve(name, style)
+    puts "Generating post sieve named #{name} with style #{style}"
+
+    file_path = "./lib/forki/scrapers/sieves/"
+    test_path = "./test/sieves/"
+    case style
+    when :video
+      test_data_valid_path = "#{test_path}/video_sieves/test_data/video_sieve_#{name}_valid.json"
+      test_data_invalid_path = "#{test_path}/video_sieves/test_data/video_sieve_#{name}_invalid.json"
+
+      file_path += "video_sieves/video_sieve_#{name}.rb"
+      test_path += "video_sieves/video_sieve_#{name}_test.rb"
+      file_template = File.read("./bin/generator_templates/video_sieve_template.rb.erb")
+      test_file_template = File.read("./bin/generator_templates/video_sieve_test_template.rb.erb")
+    when :image
+      test_data_valid_path = "#{test_path}/image_sieves/test_data/image_sieve_#{name}_valid.json"
+      test_data_invalid_path = "#{test_path}/image_sieves/test_data/image_sieve_#{name}_invalid.json"
+
+      file_path += "image_sieves/image_sieve_#{name}.rb"
+      test_path += "image_sieves/image_sieve_#{name}_test.rb"
+      file_template = File.read("./bin/generator_templates/image_sieve_template.rb.erb")
+      test_file_template = File.read("./bin/generator_templates/image_sieve_test_template.rb.erb")
+    end
+
+    file_contents = ERB.new(file_template)
+    test_file_contents = ERB.new(test_file_template)
+
+    camel_name = name.split('_').collect(&:capitalize).join
+
+    File.write(file_path, file_contents.result(binding))
+    File.write(test_path, test_file_contents.result(binding))
+
+    File.write(test_data_valid_path, "")
+    File.write(test_data_invalid_path, "")
+  end
+
+  def self.generate_user_sieve(name)
+    puts "Generating user sieve named #{name}"
+  end
+end
+
+GenerateSieve.start(ARGV)
+

data/bin/generator_templates/image_sieve_template.rb.erb

@@ -0,0 +1,63 @@
+# NOTE: This is not implemented yet, just here for filler
+
+class ImageSieve<%= camel_name %> < ImageSieve
+  # To check if it's valid for the inputted graphql objects
+  def self.check(graphql_objects)
+    image_object = self.extractor(graphql_objects)
+
+    true
+  rescue StandardError
+    return false
+  end
+
+  # output the expected format of:
+  #
+  # post_details = {
+  #   id: video_object["id"],
+  #   num_comments: num_comments,
+  #   num_shares: share_count_object.fetch("count", nil),
+  #   num_views: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["video_view_count"],
+  #   reshare_warning: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"],
+  #   video_preview_image_url: video_object["preferred_thumbnail"]["image"]["uri"],
+  #   video_url: video_object["browser_native_hd_url"] || video_object["browser_native_sd_url"],
+  #   text: text,
+  #   created_at: creation_date,
+  #   profile_link: story_node_object["comet_sections"]["context_layout"]["story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]["url"],
+  #   has_video: true
+  # }
+  # post_details[:video_preview_image_file] = Forki.retrieve_media(post_details[:video_preview_image_url])
+  # post_details[:video_file] = Forki.retrieve_media(post_details[:video_url])
+  # post_details[:reactions] = reaction_counts
+
+  def self.sieve(graphql_objects)
+    image_object = self.extractor(graphql_objects)
+
+    post_details = {
+      id: nil,
+      num_comments: nil,
+      num_shared: nil,
+      num_views: nil,
+      reshare_warning: nil,
+      video_preview_image_url: nil,
+      video_url: nil,
+      text: nil,
+      created_at: nil,
+      profile_link: nil,
+      has_video: false,
+      video_preview_image_file: Forki.retrieve_media(video_preview_image_url),
+      video_file: Forki.retrieve_media(video_url),
+      reactions: nil
+    }
+  end
+
+  private
+
+  def self.extractor(graphql_objects)
+    image_objects = graphql_objects.filter do |go|
+      # go = go.first if go.kind_of?(Array) && !go.empty?
+      # go.has_key?("image")
+    end
+
+    # image_objects.first.dig("image", "creation_story")
+  end
+end

data/bin/generator_templates/image_sieve_test_template.rb.erb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+# rubocop:disable Metrics/ClassLength
+class ImageSieve<%= camel_name %>Test < Minitest::Test
+  def setup
+    @valid_json = JSON.parse(File.read("test/sieves/image_sieves/test_data/image_sieve_<%= name %>_valid.json"))
+    @invalid_json = JSON.parse(File.read("test/sieves/image_sieves/test_data/image_sieve_<%= name %>_invalid.json"))
+  end
+
+  def test_sieve_properly_fails_check
+    assert ImageSieve<%= camel_name %>.check(@invalid_json) == false
+  end
+
+  def test_sieve_properly_passes_check
+    assert ImageSieve<%= camel_name %>.check(@valid_json)
+  end
+
+  def test_sieve_can_sieve_properly
+    result = ImageSieve<%= camel_name %>.sieve(@valid_json)
+
+    # TODO: Update the values for the post you're testing
+    # MAINTAINER TODO: FIX THIS FOR IMAGES
+    assert_equal "394367115960503", result[:id]
+    assert_equal 173, result[:num_comments]
+    assert_equal nil, result[:num_shared]
+    assert_equal nil, result[:num_views]
+    assert_equal false, result[:reshare_warning]
+    assert_not_nil result[:video_preview_image_url]
+    assert_not_nil result[:video_url]
+    assert_equal nil, text
+    assert_equal 1654989063, result[:created_at]
+    assert_equal nil, result[:profile_link]
+    assert_equal false, result[:has_video]
+    assert_not_nil result[:video_preview_image_file]
+    assert_not_nil result[:video_file]
+    assert_not_nil result[:reactions]
+
+    assert result[:reactions].kind_of?(Array)
+  end
+end

data/bin/generator_templates/video_sieve_template.rb.erb

@@ -0,0 +1,61 @@
+class VideoSieve<%= camel_name %> < VideoSieve
+  # To check if it's valid for the inputted graphql objects
+  def self.check(graphql_objects)
+    video_object = self.extractor(graphql_objects)
+
+    true
+  rescue StandardError
+    return false
+  end
+
+  # output the expected format of:
+  #
+  # post_details = {
+  #   id: video_object["id"],
+  #   num_comments: num_comments,
+  #   num_shares: share_count_object.fetch("count", nil),
+  #   num_views: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["video_view_count"],
+  #   reshare_warning: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"],
+  #   video_preview_image_url: video_object["preferred_thumbnail"]["image"]["uri"],
+  #   video_url: video_object["browser_native_hd_url"] || video_object["browser_native_sd_url"],
+  #   text: text,
+  #   created_at: creation_date,
+  #   profile_link: story_node_object["comet_sections"]["context_layout"]["story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]["url"],
+  #   has_video: true
+  # }
+  # post_details[:video_preview_image_file] = Forki.retrieve_media(post_details[:video_preview_image_url])
+  # post_details[:video_file] = Forki.retrieve_media(post_details[:video_url])
+  # post_details[:reactions] = reaction_counts
+
+  def self.sieve(graphql_objects)
+    video_object = self.extractor(graphql_objects)
+
+    post_details = {
+      id: nil,
+      num_comments: nil,
+      num_shared: nil,
+      num_views: nil,
+      reshare_warning: nil,
+      video_preview_image_url: nil,
+      video_url: nil,
+      text: nil,
+      created_at: nil,
+      profile_link: nil,
+      has_video: true,
+      video_preview_image_file: Forki.retrieve_media(video_preview_image_url),
+      video_file: Forki.retrieve_media(video_url),
+      reactions: nil
+    }
+  end
+
+  private
+
+  def self.extractor(graphql_objects)
+    video_objects = graphql_objects.filter do |go|
+      # go = go.first if go.kind_of?(Array) && !go.empty?
+      # go.has_key?("video")
+    end
+
+    # video_objects.first.dig("video", "creation_story")
+  end
+end

data/bin/generator_templates/video_sieve_test_template.rb.erb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+# rubocop:disable Metrics/ClassLength
+class VideoSieve<%= camel_name %>Test < Minitest::Test
+  def setup
+    @valid_json = JSON.parse(File.read("test/sieves/video_sieves/test_data/video_sieve_<%= name %>_valid.json"))
+    @invalid_json = JSON.parse(File.read("test/sieves/video_sieves/test_data/video_sieve_<%= name %>_invalid.json"))
+  end
+
+  def test_sieve_properly_fails_check
+    assert VideoSieve<%= camel_name %>.check(@invalid_json) == false
+  end
+
+  def test_sieve_properly_passes_check
+    assert VideoSieve<%= camel_name %>.check(@valid_json)
+  end
+
+  def test_sieve_can_sieve_properly
+    result = VideoSieve<%= camel_name %>.sieve(@valid_json)
+
+    # TODO: Update the values for the post you're testing
+    assert_equal "394367115960503", result[:id]
+    assert_equal 173, result[:num_comments]
+    assert_equal nil, result[:num_shared]
+    assert_equal nil, result[:num_views]
+    assert_equal false, result[:reshare_warning]
+    assert_not_nil result[:video_preview_image_url]
+    assert_not_nil result[:video_url]
+    assert_equal nil, text
+    assert_equal 1654989063, result[:created_at]
+    assert_equal nil, result[:profile_link]
+    assert_equal true, result[:has_video]
+    assert_not_nil result[:video_preview_image_file]
+    assert_not_nil result[:video_file]
+    assert_not_nil result[:reactions]
+
+    assert result[:reactions].kind_of?(Array)
+  end
+end

data/forki.gemspec

@@ -37,6 +37,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency "oj" # A faster JSON parser/loader than stdlib
   spec.add_dependency "selenium-webdriver" # Webdriver selenium
 
+  spec.add_development_dependency "thor" # For the generator
+
   # For more information and examples about making a new gem, checkout our
   # guide at: https://bundler.io/guides/creating_gem.html
 end

data/lib/forki/scrapers/post_scraper.rb

@@ -4,6 +4,7 @@ require "typhoeus"
 require "securerandom"
 require "byebug"
 
+
 module Forki
   # rubocop:disable Metrics/ClassLength
   class PostScraper < Scraper
@@ -100,7 +101,11 @@ module Forki
         begin
           find("span", wait: 5, text: "This Video Isn't Available Anymore", exact_text: false)
         rescue Capybara::ElementNotFound, Selenium::WebDriver::Error::StaleElementReferenceError
-          return true
+          begin
+            find("span", wait: 5, text: "This Page Isn't Available", exact_text: false)
+          rescue Capybara::ElementNotFound, Selenium::WebDriver::Error::StaleElementReferenceError
+            return true
+          end
         end
       end
 
@@ -150,21 +155,30 @@ module Forki
       end
 
       graphql_object_array = graphql_strings.map { |graphql_string| JSON.parse(graphql_string) }
+
+      # Once in awhile it's really easy
+      video_objects = graphql_object_array.filter {|go| go.has_key?("video") }
+
+      if VideoSieve.can_process_with_sieve?(graphql_object_array)
+        # Eventually all of this complexity will be replaced with this
+        return VideoSieve.sieve_for_graphql_objects(graphql_object_array)
+      end
+
       story_node_object = graphql_object_array.find { |graphql_object| graphql_object.key? "node" }&.fetch("node", nil) # user posted video
       story_node_object = story_node_object || graphql_object_array.find { |graphql_object| graphql_object.key? "nodes" }&.fetch("nodes")&.first # page posted video
 
       return extract_video_post_data_alternative(graphql_object_array) if story_node_object.nil?
 
       if story_node_object["comet_sections"]["content"]["story"]["attachments"].first["styles"]["attachment"].key?("media")
-        video_object = story_node_object["comet_sections"]["content"]["story"]["attachments"].first["styles"]["attachment"]["media"]
-        creation_date = video_object["publish_time"]
-        # creation_date = video_object["video"]["publish_time"]
+        video_object = story_node_object["comet_sections"]["content"]["story"]["attachments"].first["styles"]["attachment"]["media"]["video"]
+        creation_date = video_object["publish_time"] if video_object&.has_key("publish_time")
+        creation_date = story_node_object["comet_sections"]["content"]["story"]["attachments"].first["styles"]["attachment"]["media"] if creation_date.nil?
       elsif story_node_object["comet_sections"]["content"]["story"]["attachments"].first["styles"]["attachment"].key?("style_infos")
         # For "Reels" we need a separate way to parse this
         video_object = story_node_object["comet_sections"]["content"]["story"]["attachments"].first["styles"]["attachment"]["style_infos"].first["fb_shorts_story"]["short_form_video_context"]["playback_video"]
         creation_date = story_node_object["comet_sections"]["content"]["story"]["attachments"].first["styles"]["attachment"]["style_infos"].first["fb_shorts_story"]["creation_time"]
       else
-        raise "Unable to parse video object"
+        raise "Unable to parse video object" if video_objects.empty?
       end
 
       feedback_object = story_node_object["comet_sections"]["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]
@@ -187,7 +201,7 @@ module Forki
         num_views: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["video_view_count"],
         reshare_warning: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"],
         video_preview_image_url: video_object["preferred_thumbnail"]["image"]["uri"],
-        video_url: video_object["playable_url_quality_hd"] || video_object["playable_url"],
+        video_url: video_object["browser_native_hd_url"] || video_object["browser_native_sd_url"],
         text: text,
         created_at: creation_date,
         profile_link: story_node_object["comet_sections"]["context_layout"]["story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]["url"],
@@ -213,7 +227,7 @@ module Forki
         num_views: feedback_object["video_view_count"],
         reshare_warning: feedback_object["should_show_reshare_warning"],
         video_preview_image_url: video_object["video"]["preferred_thumbnail"]["image"]["uri"],
-        video_url: video_object["video"]["playable_url_quality_hd"] || video_object["video"]["playable_url"],
+        video_url: video_object["video"]["playable_url_quality_hd"] || video_object["video"]["browser_native_hd_url"] || video_object["video"]["browser_native_sd_url"] || video_object["video"]["playable_url"],
         text: sidepane_object["tahoe_sidepane_renderer"]["video"]["creation_story"]["comet_sections"]["message"]["story"]["message"]["text"],
         created_at: video_object["video"]["publish_time"],
         profile_link: sidepane_object["tahoe_sidepane_renderer"]["video"]["creation_story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]["url"],
@@ -365,6 +379,7 @@ module Forki
     def parse(url)
       validate_and_load_page(url)
       graphql_strings = find_graphql_data_strings(page.html)
+
       post_data = extract_post_data(graphql_strings)
       post_data[:url] = url
       user_url = post_data[:profile_link]
@@ -394,3 +409,6 @@ module Forki
     end
   end
 end
+
+require_relative "sieves/video_sieves/video_sieve"
+

data/lib/forki/scrapers/scraper.rb

@@ -7,6 +7,7 @@ require "oj"
 require "selenium-webdriver"
 require "open-uri"
 require "selenium/webdriver/remote/http/curb"
+require "cgi"
 
 options = Selenium::WebDriver::Options.chrome(exclude_switches: ["enable-automation"])
 options.add_argument("--start-maximized")
@@ -112,18 +113,41 @@ module Forki
 
       url ||= "https://www.facebook.com"
 
-
       page.driver.browser.navigate.to(url)  # Visit the url passed in or the facebook homepage if nothing is
 
       # Look for "login_form" box, which throws an error if not found. So we catch it and run the rest of the tests
       begin
         login_form = first(id: "login_form", wait: 5)
       rescue Capybara::ElementNotFound
-        return unless page.title.downcase.include?("facebook - log in")
+        begin
+          login_form = find(:xpath, '//form[@data-testid="royal_login_form"]')
+        rescue Capybara::ElementNotFound
+          return unless page.title.downcase.include?("facebook - log in")
+        end
       end
 
-      # Since we're not logged in, let's do that quick
-      page.driver.browser.navigate.to("https://www.facebook.com") if login_form.nil?
+      # Since we're not logged in, let's do that quickly
+      if login_form.nil?
+        page.driver.browser.navigate.to("https://www.facebook.com")
+
+        # Find the login form... again (Yes, we could extract this out, but it's only ever used
+        # here, so it's not worth the effort)
+        begin
+          login_form = first(id: "login_form", wait: 5)
+        rescue Capybara::ElementNotFound
+          begin
+            login_form = find(:xpath, '//form[@data-testid="royal_login_form"]')
+          rescue Capybara::ElementNotFound
+            return unless page.title.downcase.include?("facebook - log in")
+          end
+        end
+      end
+
+      if login_form.nil?
+        # maybe we're already logged in?
+        sleep(rand * 10.3)
+        return
+      end
 
       login_form.fill_in("email", with: ENV["FACEBOOK_EMAIL"])
       login_form.fill_in("pass", with: ENV["FACEBOOK_PASSWORD"])
@@ -154,13 +178,34 @@ module Forki
     # If either of those two conditions are false, raises an exception
     def validate_and_load_page(url)
       Capybara.app_host = "https://www.facebook.com"
-      facebook_url = "https://www.facebook.com"
-      # visit "https://www.facebook.com" unless current_url.start_with?(facebook_url)
-      login(url)
-      raise Forki::InvalidUrlError unless url.start_with?(facebook_url)
+      facebook_hosts = ["facebook.com", "www.facebook.com", "web.facebook.com", "m.facebook.com"]
+      parsed_url = URI.parse(url)
+      host = parsed_url.host
+      raise Forki::InvalidUrlError unless facebook_hosts.include?(host)
+
+      # Replace the host with a default one to prevent redirect loops that can happen
+      unless parsed_url.host == "www.facebook.com"
+        parsed_url.host = "www.facebook.com"
+        url = parsed_url.to_s
+      end
+
+      visit "https://www.facebook.com"
+      login
+
       visit url unless current_url.start_with?(url)
+      # # If the video is a watch page it doesn't have most of the data we want so we click on the video
+      # if url.include?("watch/live")
+      #   clickable_element = find("video")
+
+      #   while(clickable_element.obscured?)
+      #     clickable_element = clickable_element.find(:xpath, "..")
+      #   end
+
+      #   clickable_element.click
+      # end
     end
 
+
     # Extracts an integer out of a string describing a number
     # e.g. "4K Comments" returns 4000
     # e.g. "131 Shares" returns 131
@@ -171,19 +216,23 @@ module Forki
         element = element.text(:all)
       end
 
-      num_pattern = /[0-9KM ,.]+/
-      interaction_num_text = num_pattern.match(element)[0]
-
-      if interaction_num_text.include?(".")  # e.g. "2.2K"
-        interaction_num_text.to_i + interaction_num_text[-2].to_i * 100
-      elsif interaction_num_text.include?("K") # e.g. "13K"
-        interaction_num_text.to_i * 1000
-      elsif interaction_num_text.include?("M") # e.g. "13M"
-        interaction_num_text.to_i * 1_000_000
-      else  # e.g. "15,443"
-        interaction_num_text.delete!(",")
-        interaction_num_text.delete(" ").to_i
+      # Check if there's a modifier i.e. `K` or `M` if there isn't just return the number
+      unless element.include?("K") || element.include?("M")
+        element.delete(",") # "5,456" e.g.
+        return element.to_i
       end
+
+      modifier = element[-1]
+      number = element[0...-1].to_f
+
+      case modifier
+      when "K"
+        number = number * 1_000
+      when "M"
+        number = number * 1_000_000
+      end
+
+      number.to_i
     end
   end
 end

data/lib/forki/scrapers/sieves/image_sieves/image_sieve.rb.rb

@@ -0,0 +1,25 @@
+class ImageSieve
+  def self.can_process_with_sieve?(graphql_objects)
+    !sieve_class_for_graphql_objects(graphql_objects).nil?
+  end
+
+  def self.sieve_for_graphql_objects(graphql_objects)
+
+    sieve = sieve_class_for_graphql_objects(graphql_objects)
+    return nil if sieve.nil?
+
+    sieve.sieve(graphql_objects)
+  end
+
+private
+
+  def self.sieve_class_for_graphql_objects(graphql_objects)
+    sieves = []
+    sieves.detect { |sieve| sieve.check(graphql_objects) }
+  end
+end
+
+
+Dir['./lib/forki/scrapers/sieves/image_sieves/*.rb'].each do |file|
+  require file unless file.end_with?("image_sieve.rb")
+end

data/lib/forki/scrapers/sieves/video_sieves/video_sieve.rb

@@ -0,0 +1,24 @@
+class VideoSieve
+  def self.can_process_with_sieve?(graphql_objects)
+    !sieve_class_for_graphql_objects(graphql_objects).nil?
+  end
+
+  def self.sieve_for_graphql_objects(graphql_objects)
+
+    sieve = sieve_class_for_graphql_objects(graphql_objects)
+    return nil if sieve.nil?
+
+    sieve.sieve(graphql_objects)
+  end
+
+private
+
+  def self.sieve_class_for_graphql_objects(graphql_objects)
+    sieves = [VideoSieveWatchTab, VideoSieveVideoPage]
+    sieves.detect { |sieve| sieve.check(graphql_objects) }
+  end
+end
+
+Dir['./lib/forki/scrapers/sieves/video_sieves/*.rb'].each do |file|
+  require file unless file.end_with?("video_sieve.rb")
+end

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_video_page.rb

@@ -0,0 +1,66 @@
+class VideoSieveVideoPage < VideoSieve
+  # To check if it's valid for the inputted graphql objects
+  def self.check(graphql_objects)
+    story_node_object = self.extractor(graphql_objects) # This will error out
+    return false unless story_node_object["content"]["story"]["attachments"].first["styles"]["attachment"].has_key?("media")
+
+    true
+  rescue StandardError
+    return false
+  end
+
+  # output the expected format of:
+  #
+  # post_details = {
+  #   id: video_object["id"],
+  #   num_comments: num_comments,
+  #   num_shares: share_count_object.fetch("count", nil),
+  #   num_views: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["video_view_count"],
+  #   reshare_warning: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"],
+  #   video_preview_image_url: video_object["preferred_thumbnail"]["image"]["uri"],
+  #   video_url: video_object["browser_native_hd_url"] || video_object["browser_native_sd_url"],
+  #   text: text,
+  #   created_at: creation_date,
+  #   profile_link: story_node_object["comet_sections"]["context_layout"]["story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]["url"],
+  #   has_video: true
+  # }
+  # post_details[:video_preview_image_file] = Forki.retrieve_media(post_details[:video_preview_image_url])
+  # post_details[:video_file] = Forki.retrieve_media(post_details[:video_url])
+  # post_details[:reactions] = reaction_counts
+
+  def self.sieve(graphql_objects)
+    extracted_text = self.extractor(graphql_objects)
+
+    story_object = extracted_text["content"]["story"]
+    video_object = extracted_text["content"]["story"]["attachments"].first["styles"]["attachment"]["media"]
+    feedback_object = extracted_text["feedback"]["story"]["feedback_context"]["feedback_target_with_context"]["ufi_renderer"]["feedback"]["comet_ufi_summary_and_actions_renderer"]["feedback"]
+
+    video_preview_image_url = video_object["preferred_thumbnail"]["image"]["uri"]
+    video_url = video_object["browser_native_hd_url"]
+    video_url = video_object["browser_native_sd_url"] if video_url.nil?
+
+    post_details = {
+      id: video_object["id"],
+      num_comments: feedback_object["total_comment_count"],
+      num_shared: feedback_object["share_count"]["count"],
+      num_views: nil,
+      reshare_warning: feedback_object["should_show_reshare_warning"],
+      video_preview_image_url: video_preview_image_url,
+      video_url: video_url,
+      text: story_object["message"]["text"],
+      created_at: video_object["publish_time"],
+      profile_link: story_object["actors"].first["url"],
+      has_video: true,
+      video_preview_image_file: Forki.retrieve_media(video_preview_image_url),
+      video_file: Forki.retrieve_media(video_url),
+      reactions: feedback_object["cannot_see_top_custom_reactions"]["top_reactions"]["edges"]
+    }
+  end
+
+  private
+
+  def self.extractor(graphql_objects)
+    story_node_object = graphql_objects.find { |graphql_object| graphql_object.key? "node" }&.fetch("node", nil) # user posted video
+    story_node_object["comet_sections"]
+  end
+end

data/lib/forki/scrapers/sieves/video_sieves/video_sieve_watch_tab.rb

@@ -0,0 +1,91 @@
+# This is for the "watch" tab style videos https://www.facebook.com/watch/live/?v=394367115960503
+
+class VideoSieveWatchTab < VideoSieve
+  # To check if it's valid for the inputted graphql objects
+  def self.check(graphql_objects)
+    video_object = self.extractor(graphql_objects)
+    return false if video_object.nil?
+
+    video_object = video_object["attachments"]
+    return false if video_object.nil?
+
+    return false unless video_object.kind_of?(Array) && !video_object.empty?
+
+    video_object = video_object.first
+    return false unless video_object.kind_of?(Hash) && video_object.keys.include?("media")
+
+    true
+  rescue StandardError
+    return false
+  end
+
+  # output the expected format of:
+  #
+  # post_details = {
+  #   id: video_object["id"],
+  #   num_comments: num_comments,
+  #   num_shares: share_count_object.fetch("count", nil),
+  #   num_views: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["video_view_count"],
+  #   reshare_warning: feedback_object["comet_ufi_summary_and_actions_renderer"]["feedback"]["should_show_reshare_warning"],
+  #   video_preview_image_url: video_object["preferred_thumbnail"]["image"]["uri"],
+  #   video_url: video_object["browser_native_hd_url"] || video_object["browser_native_sd_url"],
+  #   text: text,
+  #   created_at: creation_date,
+  #   profile_link: story_node_object["comet_sections"]["context_layout"]["story"]["comet_sections"]["actor_photo"]["story"]["actors"][0]["url"],
+  #   has_video: true
+  # }
+  # post_details[:video_preview_image_file] = Forki.retrieve_media(post_details[:video_preview_image_url])
+  # post_details[:video_file] = Forki.retrieve_media(post_details[:video_url])
+  # post_details[:reactions] = reaction_counts
+
+  def self.sieve(graphql_objects)
+    video_object = self.extractor(graphql_objects)
+
+    video_url = video_object["attachments"].first["media"]["browser_native_sd_url"]
+    video_preview_image_url = video_object["attachments"].first["media"]["preferred_thumbnail"]["image"]["uri"]
+
+    if !video_object["feedback_context"].nil?
+      feedback_object = video_object["feedback_context"]["feedback_target_with_context"]
+    else
+      feedback_object = graphql_objects.find { |go| !go.dig("feedback", "total_comment_count").nil? }
+      feedback_object = feedback_object["feedback"] if feedback_object.has_key?("feedback")
+    end
+
+    profile_link = video_object["attachments"].first["media"]["owner"]["url"]
+    if profile_link.nil?
+      filtered_json = graphql_objects.find { |go| go.has_key? "attachments" }
+      profile_link = filtered_json["attachments"].first["media"]["creation_story"]["comet_sections"]["title"]["story"]["actors"].first["url"]
+    end
+
+    post_details = {
+      id: video_object.dig("shareable", "id") || video_object["attachments"].first["media"]["id"],
+      num_comments: feedback_object["total_comment_count"],
+      num_shared: nil, # This is not associated with these videos in this format
+      num_views: nil, # This is not associated with these videos in this format
+      reshare_warning: feedback_object["should_show_reshare_warning"],
+      video_preview_image_url: video_preview_image_url,
+      video_url: video_url,
+      text: nil, # There is no text associated with these videos
+      created_at: video_object["attachments"].first["media"]["publish_time"],
+      profile_link: profile_link,
+      has_video: true,
+      video_preview_image_file: Forki.retrieve_media(video_preview_image_url),
+      video_file: Forki.retrieve_media(video_url),
+      reactions: feedback_object["cannot_see_top_custom_reactions"]["top_reactions"]["edges"]
+    }
+  end
+
+  private
+
+  def self.extractor(graphql_objects)
+    video_objects = graphql_objects.filter do |go|
+      go = go.first if go.kind_of?(Array) && !go.empty?
+      go.has_key?("video")
+    end
+
+    story = video_objects.first.dig("video", "creation_story")
+    story = video_objects.first.dig("video", "story") if story.nil?
+
+    story
+  end
+end

data/lib/forki/scrapers/user_scraper.rb

@@ -3,10 +3,14 @@ require "typhoeus"
 module Forki
   class UserScraper < Scraper
     # Finds and returns the number of people who like the current page
-    def find_number_of_likes
-      likes_pattern = /[0-9,.KM ] people like this/
-      number_of_likes_elem = all("span").filter { | span| likes_pattern.match? span.text }.first
-      extract_int_from_num_element(number_of_likes_elem)
+    def find_number_of_likes(profile_details_string)
+      likes_pattern = /[0-9,.KM ] likes/
+      likes_pattern = /(?<num_likes>[0-9,.KM ]+) (l|L)ikes/
+      number_of_likes_match = likes_pattern.match(profile_details_string)
+
+      return nil if number_of_likes_match.nil?
+
+      extract_int_from_num_element(number_of_likes_match.named_captures["num_likes"])
     end
 
     # Finds and returns the number of people who follow the current page
@@ -14,8 +18,18 @@ module Forki
       followers_pattern = /Followed by (?<num_followers>[0-9,.KM ]) people/
       alt_follower_pattern = /(?<num_followers>[0-9,.KM ]+) (f|F)ollowers/
       number_of_followers_match = followers_pattern.match(profile_details_string) || alt_follower_pattern.match(profile_details_string)
+
       return nil if number_of_followers_match.nil?
-      extract_int_from_num_element(number_of_followers_match.named_captures["num_followers"])
+
+      number_of_followers = extract_int_from_num_element(number_of_followers_match.named_captures["num_followers"])
+
+      # Note, this is sticking around if we want to use it later
+      # if number_of_followers.nil?
+      #   number_of_followers_string = JSON.parse(profile_header_str)["user"]["profile_header_renderer"]["user"]["profile_social_context"]["content"].first["text"]["text"]
+      #   number_of_followers = extract_int_from_num_element(number_of_followers_string)
+      # end
+
+      number_of_followers
     end
 
     def find_number_followers_for_normal_profile(profile_followers_node)
@@ -61,6 +75,7 @@ module Forki
         verified: profile_header_obj["user"]["is_verified"],
         profile: profile_intro_obj ? profile_intro_obj["profile_intro_card"]["bio"]["text"] : "",
         profile_image_url: profile_header_obj["user"]["profilePicLarge"]["uri"],
+        number_of_likes: find_number_of_likes(profile_header_str),
       }
     end
 

data/lib/forki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Forki
-  VERSION = "0.1.3"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forki
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - ''
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-05 00:00:00.000000000 Z
+date: 2023-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description:
 email:
 - ''
@@ -99,12 +113,21 @@ files:
 - README.md
 - Rakefile
 - bin/console
+- bin/generate_sieve
+- bin/generator_templates/image_sieve_template.rb.erb
+- bin/generator_templates/image_sieve_test_template.rb.erb
+- bin/generator_templates/video_sieve_template.rb.erb
+- bin/generator_templates/video_sieve_test_template.rb.erb
 - bin/setup
 - forki.gemspec
 - lib/forki.rb
 - lib/forki/post.rb
 - lib/forki/scrapers/post_scraper.rb
 - lib/forki/scrapers/scraper.rb
+- lib/forki/scrapers/sieves/image_sieves/image_sieve.rb.rb
+- lib/forki/scrapers/sieves/video_sieves/video_sieve.rb
+- lib/forki/scrapers/sieves/video_sieves/video_sieve_video_page.rb
+- lib/forki/scrapers/sieves/video_sieves/video_sieve_watch_tab.rb
 - lib/forki/scrapers/user_scraper.rb
 - lib/forki/user.rb
 - lib/forki/version.rb
@@ -139,7 +162,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.4.14
 signing_key:
 specification_version: 4
 summary: A gem to scrape Facebook pages for archive purposes.