zorki 0.1.26 → 0.1.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec1dba1c80d66f33d1e50fc104511a5842091529c6d2b1e3f4758508ae37468a
-  data.tar.gz: b510d48cbc12d0e9b70fb17fb91aeffdad51d85f85e5813695df1c86279894db
+  metadata.gz: c5472c0d436e13f2e8554b59051546fed9400ad793de71b9b2d546bb5bd02d08
+  data.tar.gz: d62650105cb0f41a48a93d4379e077a4c1b658e96ae13a30c1d8073f8f2e0546
 SHA512:
-  metadata.gz: 6463fc59b818e21d4c515212d04549b7a3b925433d7e9df79ff376a786f74e028ab99146b8422c64384470443d88835854717e3a19a7c68bb3a767e7277f1c39
-  data.tar.gz: 173b570bb5eb62d759a488c21aff0092c633db1f0f1a4fa771e46ad32141a2016386f698e724cbc1a7dced5ad66a9d7b22add8f7dd533d8dc88ec009ca9a2814
+  metadata.gz: 84a98236f4ca36daf440a8aea29acec2fa6963508bae78f5ee7c4d92c2ffedf19ef8db4050deadaa5090ea770132d2a47c64a1bab87f52329bdf18dd31f4aa2e
+  data.tar.gz: e1b635b352163d08dc0ea9b5e74b3cb990a4f9a7d91ce29296ae2150692612c2a7a81fc9e04bfd33cedfd5c4dab7031e5f06a802a25032aed15036550f306328

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    zorki (0.1.25)
+    zorki (0.1.26)
       apparition
       capybara
       oj

data/lib/zorki/scrapers/scraper.rb

@@ -51,8 +51,13 @@ module Zorki
     # additional_search_params is a comma seperated keys
     # example: `data,xdt_api__v1__media__shortcode__web_info,items`
     #
+    # NOTE: `post_data_include` if not nil overrules the additional_search_parameters
+    # This is so that i didn't have to refactor the entire code base when I added it.
+    # Eventually it might be better to look at the post request and see if we can do the
+    # same type of search there as we use for users and simplify this whole thing a lot.
+    #
     # @returns Hash a ruby hash of the JSON data
-    def get_content_of_subpage_from_url(url, subpage_search, additional_search_parameters = nil)
+    def get_content_of_subpage_from_url(url, subpage_search, additional_search_parameters = nil, post_data_include: nil)
       # So this is fun:
       # For pages marked as misinformation we have to use one method (interception of requrest) and
       # for pages that are not, we can just pull the data straight from the page.
@@ -65,21 +70,28 @@ module Zorki
       # the one we want, and then moves on.
       response_body = nil
 
+      responses = []
       page.driver.browser.intercept do |request, &continue|
         # This passes the request forward unmodified, since we only care about the response
-        # puts "checking request: #{request.url}"
+        #
+        # responses.first.post_data.include?("render_surface%22%3A%22PROFILE")
         continue.call(request) && next unless request.url.include?(subpage_search)
+        continue.call(request) && next unless !post_data_include.nil? && request.post_data.include?(post_data_include)
 
         continue.call(request) do |response|
+          puts "***********************************************************"
+          puts "checking request: #{request.url}"
+          puts response.body
+          puts "***********************************************************"
+
+          # responses << response
           # Check if not a CORS prefetch and finish up if not
           if !response.body&.empty? && response.body
             check_passed = true
 
-            unless additional_search_parameters.nil?
+            if !additional_search_parameters.nil? && post_data_include.nil?
               body_to_check = Oj.load(response.body)
 
-              debugger if body_to_check.include?("jokoy.komi.io")
-
               search_parameters = additional_search_parameters.split(",")
               search_parameters.each_with_index do |key, index|
                 break if body_to_check.nil?
@@ -89,6 +101,13 @@ module Zorki
               end
             end
 
+            if check_passed == false
+              puts "***********************************************************"
+              puts "checking FAILED request: #{request.url}"
+              puts response.body
+              puts "***********************************************************"
+            end
+
             response_body = response.body if check_passed == true
           end
         end
@@ -120,17 +139,17 @@ module Zorki
         elements = doc.search("script").filter_map do |element|
           parsed_element_json = nil
           begin
-            element_json = JSON.parse(element.text)
+            element_json = OJ.load(element.text)
 
-            if element.text.include?("jokoy.komi.io")
-              debugger
+            # if element.text.include?("jokoy.komi.io")
+              # debugger
               # if element_json["require"].first.last.first["__bbox"].key?("require")
 
               #   element_json["require"].first.last.first["__bbox"]["require"].each do |x|
               #     debugger if x.to_s.include?("Si mulut pelaut")
               #   end
               # end
-            end
+            # end
 
             parsed_element_json = element_json["require"].last.last.first["__bbox"]["require"].first.last.last["__bbox"]["result"]["data"]["xdt_api__v1__media__shortcode__web_info"]
           rescue StandardError

data/lib/zorki/scrapers/user_scraper.rb

@@ -20,13 +20,11 @@ module Zorki
       graphql_script = nil
       count = 0
       loop do
-        raise Zorki::UserScrapingError.new("Zorki could not find user #{username}", additional_data: { username: username }) if count > 3
-
         print "Scraping user #{username}... (attempt #{count + 1})\n"
         begin
           login
 
-          graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "graphql", "data,user,media_count")
+          graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "graphql/query", "data,user,media_count", post_data_include: "render_surface")
           graphql_script = graphql_script.first if graphql_script.class == Array
 
           if graphql_script.nil?
@@ -34,8 +32,13 @@ module Zorki
           end
         rescue Zorki::ContentUnavailableError => e
           count += 1
+
+          if count > 3
+            raise Zorki::UserScrapingError.new("Zorki could not find user #{username}", additional_data: { username: username })
+          end
+
           page.driver.browser.navigate.to("https://www.instagram.com") # we want to go back to the main page so we start from scratch
-          sleep 10
+          sleep rand(5..10)
           next
         end
 

data/lib/zorki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zorki
-  VERSION = "0.1.26"
+  VERSION = "0.1.27"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zorki
 version: !ruby/object:Gem::Version
-  version: 0.1.26
+  version: 0.1.27
 platform: ruby
 authors:
 - Christopher Guess
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-09-16 00:00:00.000000000 Z
+date: 2024-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara