zorki 0.2.4 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 92619beb5d1040cbab5fd2733fb9bf0d61260e424137ac380f43f6b9b1abc560
-  data.tar.gz: d50ab33a23e01277ae94149ea1bce2758c4fe54e64aa9a301ac6fb84069c92ee
+  metadata.gz: 19f084d94393dd8e69f559d7f9bc84385d80b1ac2730c0e957f031d5e67da38f
+  data.tar.gz: f15fbb5c622bcd2940a1adda2748178d1314ac579e31b94d859979c6e507df76
 SHA512:
-  metadata.gz: 92ee5f59ed8c65667ed250609f69dc4fd8163797a6b71679baaa758b40e39d3835398a813ace58b4b26019ed1675b81bc6c780a8e38b443d5344826ece822600
-  data.tar.gz: a9f8b0258cef20f4c436732f988a2bcf0c44334753eacc9f2684d4e0b3b44cf98ef62269139f4261bbd49be22c8151c5e97bf552f30845d149a34c7055a3a5c9
+  metadata.gz: ea049add265fd88524d894995f5cc3eabdf98c6b9610969ff7344dc86ce2e679ac0f2d6a5e9f606b2111806789d6a783c23dff9596f8a7dc66be8543eb8a3453
+  data.tar.gz: e6d9760c423d59280cee47262d2d92e50312c0b858ee9de2dbf4aafa32ee1cb5d7634b3a55ff2b36b5fadb2be8cc52f3b63190239a61c2f1b03116bd3a83c75f

data/lib/zorki/scrapers/post_scraper.rb

@@ -135,9 +135,21 @@ module Zorki
         # Go through the entire JSON structure (below for now) and make sure it hits all the points
 
         object = graphql_object["data"]["xdt_shortcode_media"]
-        date = object["edge_media_to_caption"]["edges"].first["node"]["created_at"]
+
+        begin
+          date = object["edge_media_to_caption"]["edges"].first["node"]["created_at"]
+        rescue StandardError
+          date = object["taken_at_timestamp"].to_s
+        end
+
         date = DateTime.strptime(date, "%s")
-        text = object["edge_media_to_caption"]["edges"].first["node"]["text"]
+
+        begin
+          text = object["edge_media_to_caption"]["edges"].first["node"]["text"]
+        rescue StandardError
+          text = ""
+        end
+
         number_of_likes = object["edge_media_preview_like"]["count"]
         username = object["owner"]["username"]
         id = object["shortcode"]
@@ -159,7 +171,7 @@ module Zorki
           end
         elsif object.has_key?("display_resources")
           if object["is_video"] == true
-            video = Zorki.retrieve_media(object["display_resources"].last["src"])
+            video = Zorki.retrieve_media(object["video_url"])
             video_preview_image = Zorki.retrieve_media(object["display_url"])
           else
             images << Zorki.retrieve_media(object["display_resources"].last["src"])

data/lib/zorki/scrapers/scraper.rb

@@ -57,7 +57,7 @@ module Zorki
     # same type of search there as we use for users and simplify this whole thing a lot.
     #
     # @returns Hash a ruby hash of the JSON data
-    def get_content_of_subpage_from_url(url, subpage_search, additional_search_parameters = nil, post_data_include: nil)
+    def get_content_of_subpage_from_url(url, subpage_search, additional_search_parameters = nil, post_data_include: nil, header: nil)
       # So this is fun:
       # For pages marked as misinformation we have to use one method (interception of requrest) and
       # for pages that are not, we can just pull the data straight from the page.
@@ -73,25 +73,22 @@ module Zorki
       page.driver.browser.intercept do |request, &continue|
         # This passes the request forward unmodified, since we only care about the response
         continue.call(request) && next unless request.url.include?(subpage_search)
-        continue.call(request) && next unless !post_data_include.nil? && request.post_data&.include?(post_data_include)
+        if !header.nil?
+          header_key = header.keys.first.to_s
+          header_value = header.values.first
+
+          puts "Request Header included? #{request.headers.include?(header_key)} #{request.headers[header_key]} == #{header_value}"
+          continue.call(request) && next unless request.headers.include?(header_key) && request.headers[header_key] == header_value
+
+        elsif !post_data_include.nil?
+          continue.call(request) && next unless request.post_data&.include?(post_data_include)
+        end
 
         continue.call(request) do |response|
           # Check if not a CORS prefetch and finish up if not
           if !response.body&.empty? && response.body
             check_passed = true
 
-            if !additional_search_parameters.nil? && post_data_include.nil?
-              body_to_check = Oj.load(response.body)
-
-              search_parameters = additional_search_parameters.split(",")
-              search_parameters.each_with_index do |key, index|
-                break if body_to_check.nil?
-
-                check_passed = false unless body_to_check.has_key?(key)
-                body_to_check = body_to_check[key]
-              end
-            end
-
             next if check_passed == false
             response_body = response.body if check_passed == true
           end
@@ -108,42 +105,17 @@ module Zorki
       page.driver.browser.navigate.to(url)
       # We wait until the correct intercept is processed or we've waited 60 seconds
       start_time = Time.now
-      # puts "Waiting.... #{url}"
-
-      sleep(rand(1...10))
       while response_body.nil? && (Time.now - start_time) < 60
         sleep(0.1)
       end
 
       page.driver.execute_script("window.stop();")
 
-      # If this is a page that has not been marked as misinfo we can just pull the data
-      # TODO: put this before the whole load loop
-      if response_body.nil?
-        doc = Nokogiri::HTML(page.driver.browser.page_source)
-        # elements = doc.search("script").find_all do |e|
-        #   e.attributes.has_key?("type") && e.attributes["type"].value == "application/ld+json"
-        # end
-
-        elements = doc.search("script").filter_map do |element|
-          parsed_element_json = nil
-          begin
-            element_json = Oj.load(element.text)
-            parsed_element_json = element_json["require"].last.last.first["__bbox"]["require"].first.last.last["__bbox"]["result"]["data"]["xdt_api__v1__media__shortcode__web_info"]
-          rescue StandardError
-            next
-          end
-
-          parsed_element_json
-        end
-
-        if elements&.empty?
-          raise ContentUnavailableError.new("Cannot find anything", additional_data: { page_source: page.driver.browser.page_source, elements: elements })
-        end
-
-        return elements
-      end
+      # 1. Fix the ability to dettect if a page is removed -DONE
+      # 2. Fix videos for slideshows - Works for reels?
+      # 3. Public liinks
 
+      # Check if something failed before we continue. Use the fake test to test
       raise ContentUnavailableError.new("Response body nil") if response_body.nil?
       Oj.load(response_body)
     ensure

data/lib/zorki/scrapers/user_scraper.rb

@@ -26,7 +26,7 @@ module Zorki
 
           # This is searching for a specific request, the reason it's weird is because it's uri encoded
           # graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "graphql/query", "data,user,media_count", post_data_include: "render_surface%22%3A%22PROFILE")
-          graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "graphql/query", nil, post_data_include: "render_surface%22%3A%22PROFILE")
+          graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "graphql/query", nil, post_data_include: "render_surface%22%3A%22PROFILE", header: { "X-FB-Friendly-Name": "PolarisProfilePageContentQuery" })
           graphql_script = graphql_script.first if graphql_script.class == Array
 
           if graphql_script.nil?

data/lib/zorki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zorki
-  VERSION = "0.2.4"
+  VERSION = "0.2.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zorki
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.6
 platform: ruby
 authors:
 - Christopher Guess
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-17 00:00:00.000000000 Z
+date: 2024-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara