zorki 0.2.5 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5d6f47b685aa9f49c602bc0082e7ac9084e0cb444acfa6a7989c5d5cd3d572e7
-  data.tar.gz: 858f01c64e0efa666941222d0fade285f0c49843957cdbc0adc0dbe46e7c7ec0
+  metadata.gz: 47ee4ce0e4bc429103d8086157b53e1db6d21d8ecaf50872913d52fe18e76f0d
+  data.tar.gz: 930e34afa3d53c82991e2deddc3a3e5d6f7062a7119bc8a86787a98e7c42c16f
 SHA512:
-  metadata.gz: 73a9ac083a40f2e7c8b03a315bd50b83545507ed9e1700d944fa8e1e0e781ce8cad6ff5a17e277d83bd1788cd7444774c5ae530be8c3205213c7dfcbfb27c9c4
-  data.tar.gz: 201bc5e5eab638249bd4fb1e5dd859ed308e1e6fdbc4438d709144aa519bcef393404a8206929636944f65217af7e8597a6b8589f1a86f16ea3c6513e5806487
+  metadata.gz: 6abd258c185bd5817d97d29baf9be8751cf970e52d5431bf3dae890adbf153bcc9410487698f5e4d6b260fe3d68fb78c81704817480faec2f770c598c23c60cd
+  data.tar.gz: 8ffee00d7d7e505d4f5c84193463737ffe58a5d81a26ceb3778f0712f79e77662059445a39d152e166bf554919610120a8c9f36e0f87cce43032f4db8e62d133

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    zorki (0.1.26)
+    zorki (0.2.6)
       apparition
       capybara
       oj
@@ -87,7 +87,7 @@ GEM
     regexp_parser (2.9.2)
     reline (0.5.10)
       io-console (~> 0.5)
-    rexml (3.3.7)
+    rexml (3.3.8)
     rubocop (1.66.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
@@ -125,7 +125,7 @@ GEM
     ruby-progressbar (1.13.0)
     rubyzip (2.3.2)
     securerandom (0.3.1)
-    selenium-devtools (0.128.0)
+    selenium-devtools (0.129.0)
       selenium-webdriver (~> 4.2)
     selenium-webdriver (4.24.0)
       base64 (~> 0.2)

data/lib/zorki/scrapers/post_scraper.rb

@@ -171,7 +171,7 @@ module Zorki
           end
         elsif object.has_key?("display_resources")
           if object["is_video"] == true
-            video = Zorki.retrieve_media(object["display_resources"].last["src"])
+            video = Zorki.retrieve_media(object["video_url"])
             video_preview_image = Zorki.retrieve_media(object["display_url"])
           else
             images << Zorki.retrieve_media(object["display_resources"].last["src"])

data/lib/zorki/scrapers/scraper.rb

@@ -57,7 +57,7 @@ module Zorki
     # same type of search there as we use for users and simplify this whole thing a lot.
     #
     # @returns Hash a ruby hash of the JSON data
-    def get_content_of_subpage_from_url(url, subpage_search, additional_search_parameters = nil, post_data_include: nil)
+    def get_content_of_subpage_from_url(url, subpage_search, additional_search_parameters = nil, post_data_include: nil, header: nil)
       # So this is fun:
       # For pages marked as misinformation we have to use one method (interception of requrest) and
       # for pages that are not, we can just pull the data straight from the page.
@@ -73,25 +73,22 @@ module Zorki
       page.driver.browser.intercept do |request, &continue|
         # This passes the request forward unmodified, since we only care about the response
         continue.call(request) && next unless request.url.include?(subpage_search)
-        continue.call(request) && next unless !post_data_include.nil? && request.post_data&.include?(post_data_include)
+        if !header.nil?
+          header_key = header.keys.first.to_s
+          header_value = header.values.first
+
+          puts "Request Header included? #{request.headers.include?(header_key)} #{request.headers[header_key]} == #{header_value}"
+          continue.call(request) && next unless request.headers.include?(header_key) && request.headers[header_key] == header_value
+
+        elsif !post_data_include.nil?
+          continue.call(request) && next unless request.post_data&.include?(post_data_include)
+        end
 
         continue.call(request) do |response|
           # Check if not a CORS prefetch and finish up if not
           if !response.body&.empty? && response.body
             check_passed = true
 
-            if !additional_search_parameters.nil? && post_data_include.nil?
-              body_to_check = Oj.load(response.body)
-
-              search_parameters = additional_search_parameters.split(",")
-              search_parameters.each_with_index do |key, index|
-                break if body_to_check.nil?
-
-                check_passed = false unless body_to_check.has_key?(key)
-                body_to_check = body_to_check[key]
-              end
-            end
-
             next if check_passed == false
             response_body = response.body if check_passed == true
           end
@@ -108,42 +105,17 @@ module Zorki
       page.driver.browser.navigate.to(url)
       # We wait until the correct intercept is processed or we've waited 60 seconds
       start_time = Time.now
-      # puts "Waiting.... #{url}"
-
-      sleep(rand(1...10))
       while response_body.nil? && (Time.now - start_time) < 60
         sleep(0.1)
       end
 
       page.driver.execute_script("window.stop();")
 
-      # If this is a page that has not been marked as misinfo we can just pull the data
-      # TODO: put this before the whole load loop
-      if response_body.nil?
-        doc = Nokogiri::HTML(page.driver.browser.page_source)
-        # elements = doc.search("script").find_all do |e|
-        #   e.attributes.has_key?("type") && e.attributes["type"].value == "application/ld+json"
-        # end
-
-        elements = doc.search("script").filter_map do |element|
-          parsed_element_json = nil
-          begin
-            element_json = Oj.load(element.text)
-            parsed_element_json = element_json["require"].last.last.first["__bbox"]["require"].first.last.last["__bbox"]["result"]["data"]["xdt_api__v1__media__shortcode__web_info"]
-          rescue StandardError
-            next
-          end
-
-          parsed_element_json
-        end
-
-        if elements&.empty?
-          raise ContentUnavailableError.new("Cannot find anything", additional_data: { page_source: page.driver.browser.page_source, elements: elements })
-        end
-
-        return elements
-      end
+      # 1. Fix the ability to dettect if a page is removed -DONE
+      # 2. Fix videos for slideshows - Works for reels?
+      # 3. Public liinks
 
+      # Check if something failed before we continue. Use the fake test to test
       raise ContentUnavailableError.new("Response body nil") if response_body.nil?
       Oj.load(response_body)
     ensure

data/lib/zorki/scrapers/user_scraper.rb

@@ -26,7 +26,7 @@ module Zorki
 
           # This is searching for a specific request, the reason it's weird is because it's uri encoded
           # graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "graphql/query", "data,user,media_count", post_data_include: "render_surface%22%3A%22PROFILE")
-          graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "graphql/query", nil, post_data_include: "render_surface%22%3A%22PROFILE")
+          graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "graphql/query", nil, post_data_include: "render_surface%22%3A%22PROFILE", header: { "X-FB-Friendly-Name": "PolarisProfilePageContentQuery" })
           graphql_script = graphql_script.first if graphql_script.class == Array
 
           if graphql_script.nil?

data/lib/zorki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zorki
-  VERSION = "0.2.5"
+  VERSION = "0.2.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zorki
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.2.7
 platform: ruby
 authors:
 - Christopher Guess
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-17 00:00:00.000000000 Z
+date: 2024-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara