zorki 0.1.24 → 0.1.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa95bb5ca131ca6b7faed2aab200329579b77012cb079da75a0ad90a60daa5bd
-  data.tar.gz: 82eb0cc29af2cfaeafa8774e027904959f953a50faa03d71dbcbeba9595ac520
+  metadata.gz: c5472c0d436e13f2e8554b59051546fed9400ad793de71b9b2d546bb5bd02d08
+  data.tar.gz: d62650105cb0f41a48a93d4379e077a4c1b658e96ae13a30c1d8073f8f2e0546
 SHA512:
-  metadata.gz: 127638ce83ed09be71f194ba3e5dc269374890a1bd89874c8f158896a7664d41724698a258916f63e373b1c6f3dd3b4d315fac49a15e17ff457035ec4d345120
-  data.tar.gz: 105b7e148774a82640ed48bffe46b21d941ce0a18987f59ef0e79a3a058077fbd9dda651811bbc421ff9fcc8323de152b2ac5afbff0e5b9d4e5c65d27f5ac85c
+  metadata.gz: 84a98236f4ca36daf440a8aea29acec2fa6963508bae78f5ee7c4d92c2ffedf19ef8db4050deadaa5090ea770132d2a47c64a1bab87f52329bdf18dd31f4aa2e
+  data.tar.gz: e1b635b352163d08dc0ea9b5e74b3cb990a4f9a7d91ce29296ae2150692612c2a7a81fc9e04bfd33cedfd5c4dab7031e5f06a802a25032aed15036550f306328

data/Gemfile.lock

@@ -1,35 +1,36 @@
 PATH
   remote: .
   specs:
-    zorki (0.1.9)
+    zorki (0.1.26)
       apparition
       capybara
       oj
       selenium-devtools
-      selenium-webdriver (~> 4.19)
+      selenium-webdriver (~> 4.24.0)
       typhoeus
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.1.3.2)
+    activesupport (7.2.1)
       base64
       bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+      concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
       drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      mutex_m
-      tzinfo (~> 2.0)
-    addressable (2.8.6)
-      public_suffix (>= 2.0.2, < 6.0)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+    addressable (2.8.7)
+      public_suffix (>= 2.0.2, < 7.0)
     apparition (0.6.0)
       capybara (~> 3.13, < 4)
       websocket-driver (>= 0.6.5)
     ast (2.4.2)
     base64 (0.2.0)
-    bigdecimal (3.1.7)
+    bigdecimal (3.1.8)
     capybara (3.40.0)
       addressable
       matrix
@@ -39,75 +40,76 @@ GEM
       rack-test (>= 0.6.3)
       regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
-    concurrent-ruby (1.2.3)
+    concurrent-ruby (1.3.4)
     connection_pool (2.4.1)
-    curb (1.0.5)
-    debug (1.9.1)
+    curb (1.0.6)
+    debug (1.9.2)
       irb (~> 1.10)
       reline (>= 0.3.8)
     dotenv (2.7.6)
     drb (2.2.1)
     ethon (0.16.0)
       ffi (>= 1.15.0)
-    ffi (1.16.3)
-    i18n (1.14.4)
+    ffi (1.17.0-arm64-darwin)
+    i18n (1.14.5)
       concurrent-ruby (~> 1.0)
     io-console (0.7.2)
-    irb (1.12.0)
-      rdoc
+    irb (1.14.0)
+      rdoc (>= 4.0.0)
       reline (>= 0.4.2)
-    json (2.7.1)
+    json (2.7.2)
     language_server-protocol (3.17.0.3)
+    logger (1.6.1)
     matrix (0.4.2)
     mini_mime (1.1.5)
-    minitest (5.22.3)
-    mutex_m (0.2.0)
-    nokogiri (1.16.3-arm64-darwin)
+    minitest (5.25.1)
+    nokogiri (1.16.7-arm64-darwin)
       racc (~> 1.4)
-    oj (3.16.3)
+    oj (3.16.6)
       bigdecimal (>= 3.0)
-    parallel (1.24.0)
-    parser (3.3.0.5)
+      ostruct (>= 0.2)
+    ostruct (0.6.0)
+    parallel (1.26.3)
+    parser (3.3.5.0)
       ast (~> 2.4.1)
       racc
     psych (5.1.2)
       stringio
-    public_suffix (5.0.4)
-    racc (1.7.3)
+    public_suffix (6.0.1)
+    racc (1.8.1)
     rack (3.0.8)
     rack-test (2.1.0)
       rack (>= 1.3)
     rainbow (3.1.1)
-    rake (13.1.0)
-    rdoc (6.6.3.1)
+    rake (13.2.1)
+    rdoc (6.7.0)
       psych (>= 4.0.0)
-    regexp_parser (2.9.0)
-    reline (0.5.0)
+    regexp_parser (2.9.2)
+    reline (0.5.10)
       io-console (~> 0.5)
-    rexml (3.2.6)
-    rubocop (1.62.1)
+    rexml (3.3.7)
+    rubocop (1.66.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.31.1, < 2.0)
+      regexp_parser (>= 2.4, < 3.0)
+      rubocop-ast (>= 1.32.2, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.31.2)
-      parser (>= 3.3.0.4)
-    rubocop-md (1.2.2)
-      rubocop (>= 1.0)
-    rubocop-minitest (0.35.0)
+    rubocop-ast (1.32.3)
+      parser (>= 3.3.1.0)
+    rubocop-md (1.2.3)
+      rubocop (>= 1.45)
+    rubocop-minitest (0.36.0)
       rubocop (>= 1.61, < 2.0)
       rubocop-ast (>= 1.31.1, < 2.0)
     rubocop-packaging (0.5.2)
       rubocop (>= 1.33, < 2.0)
-    rubocop-performance (1.20.2)
+    rubocop-performance (1.21.1)
       rubocop (>= 1.48.1, < 2.0)
-      rubocop-ast (>= 1.30.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
     rubocop-rails (2.19.1)
       activesupport (>= 4.2.0)
       rack (>= 1.1)
@@ -122,20 +124,22 @@ GEM
       rubocop-rails (~> 2.0)
     ruby-progressbar (1.13.0)
     rubyzip (2.3.2)
-    selenium-devtools (0.123.0)
+    securerandom (0.3.1)
+    selenium-devtools (0.128.0)
       selenium-webdriver (~> 4.2)
-    selenium-webdriver (4.19.0)
+    selenium-webdriver (4.24.0)
       base64 (~> 0.2)
+      logger (~> 1.4)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
-    stringio (3.1.0)
+    stringio (3.1.1)
     typhoeus (1.4.1)
       ethon (>= 0.9.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.5.0)
-    websocket (1.2.10)
+    unicode-display_width (2.6.0)
+    websocket (1.2.11)
     websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)

data/lib/zorki/scrapers/scraper.rb

@@ -51,8 +51,13 @@ module Zorki
     # additional_search_params is a comma seperated keys
     # example: `data,xdt_api__v1__media__shortcode__web_info,items`
     #
+    # NOTE: `post_data_include` if not nil overrules the additional_search_parameters
+    # This is so that i didn't have to refactor the entire code base when I added it.
+    # Eventually it might be better to look at the post request and see if we can do the
+    # same type of search there as we use for users and simplify this whole thing a lot.
+    #
     # @returns Hash a ruby hash of the JSON data
-    def get_content_of_subpage_from_url(url, subpage_search, additional_search_parameters = nil)
+    def get_content_of_subpage_from_url(url, subpage_search, additional_search_parameters = nil, post_data_include: nil)
       # So this is fun:
       # For pages marked as misinformation we have to use one method (interception of requrest) and
       # for pages that are not, we can just pull the data straight from the page.
@@ -65,21 +70,28 @@ module Zorki
       # the one we want, and then moves on.
       response_body = nil
 
+      responses = []
       page.driver.browser.intercept do |request, &continue|
         # This passes the request forward unmodified, since we only care about the response
-        # puts "checking request: #{request.url}"
+        #
+        # responses.first.post_data.include?("render_surface%22%3A%22PROFILE")
         continue.call(request) && next unless request.url.include?(subpage_search)
+        continue.call(request) && next unless !post_data_include.nil? && request.post_data.include?(post_data_include)
 
         continue.call(request) do |response|
+          puts "***********************************************************"
+          puts "checking request: #{request.url}"
+          puts response.body
+          puts "***********************************************************"
+
+          # responses << response
           # Check if not a CORS prefetch and finish up if not
           if !response.body&.empty? && response.body
             check_passed = true
 
-            unless additional_search_parameters.nil?
+            if !additional_search_parameters.nil? && post_data_include.nil?
               body_to_check = Oj.load(response.body)
 
-              debugger if body_to_check.include?("jokoy.komi.io")
-
               search_parameters = additional_search_parameters.split(",")
               search_parameters.each_with_index do |key, index|
                 break if body_to_check.nil?
@@ -89,6 +101,13 @@ module Zorki
               end
             end
 
+            if check_passed == false
+              puts "***********************************************************"
+              puts "checking FAILED request: #{request.url}"
+              puts response.body
+              puts "***********************************************************"
+            end
+
             response_body = response.body if check_passed == true
           end
         end
@@ -120,17 +139,17 @@ module Zorki
         elements = doc.search("script").filter_map do |element|
           parsed_element_json = nil
           begin
-            element_json = JSON.parse(element.text)
+            element_json = OJ.load(element.text)
 
-            if element.text.include?("jokoy.komi.io")
-              debugger
+            # if element.text.include?("jokoy.komi.io")
+              # debugger
               # if element_json["require"].first.last.first["__bbox"].key?("require")
 
               #   element_json["require"].first.last.first["__bbox"]["require"].each do |x|
               #     debugger if x.to_s.include?("Si mulut pelaut")
               #   end
               # end
-            end
+            # end
 
             parsed_element_json = element_json["require"].last.last.first["__bbox"]["require"].first.last.last["__bbox"]["result"]["data"]["xdt_api__v1__media__shortcode__web_info"]
           rescue StandardError
@@ -141,12 +160,14 @@ module Zorki
         end
 
         if elements&.empty?
+          # debugger
           raise ContentUnavailableError.new("Cannot find anything", additional_data: { page_source: page.driver.browser.page_source, elements: elements })
         end
 
         return elements
       end
 
+      # debugger if response_body.nil?
       raise ContentUnavailableError.new("Response body nil") if response_body.nil?
       Oj.load(response_body)
     ensure

data/lib/zorki/scrapers/user_scraper.rb

@@ -16,13 +16,33 @@ module Zorki
       #   - *description
       #   - *links
       # - *Profile image
-      login
 
-      graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "graphql", "data,user,full_name")
-      graphql_script = graphql_script.first if graphql_script.class == Array
+      graphql_script = nil
+      count = 0
+      loop do
+        print "Scraping user #{username}... (attempt #{count + 1})\n"
+        begin
+          login
+
+          graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "graphql/query", "data,user,media_count", post_data_include: "render_surface")
+          graphql_script = graphql_script.first if graphql_script.class == Array
+
+          if graphql_script.nil?
+            graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "web_profile_info")
+          end
+        rescue Zorki::ContentUnavailableError => e
+          count += 1
+
+          if count > 3
+            raise Zorki::UserScrapingError.new("Zorki could not find user #{username}", additional_data: { username: username })
+          end
 
-      if graphql_script.nil?
-        graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "web_profile_info")
+          page.driver.browser.navigate.to("https://www.instagram.com") # we want to go back to the main page so we start from scratch
+          sleep rand(5..10)
+          next
+        end
+
+        break
       end
 
       if graphql_script.has_key?("author") && !graphql_script["author"].nil?
@@ -36,10 +56,14 @@ module Zorki
           ["https://schema.org/FilmAction", "http://schema.org/WriteAction"].include?(stat["interactionType"])
         end.first
 
+        # number_of_posts = graphql_script["data"]["user"]["media_count"] if number_of_posts.nil?
+
         number_of_followers = graphql_script["interactionStatistic"].select do |stat|
           stat["interactionType"] == "http://schema.org/FollowAction"
         end.first
 
+        # number_of_followers = graphql_script["data"]["user"]["follower_count"] if number_of_followers.nil?
+
         begin
           profile_image_url = user["image"]
           {
@@ -76,7 +100,8 @@ module Zorki
           profile_image_url: profile_image_url
         }
       end
-    rescue Zorki::ContentUnavailableError
+    rescue Zorki::ContentUnavailableError => e
+      debugger
       raise Zorki::UserScrapingError.new("Zorki could not find user #{username}", additional_data: { username: username })
     end
   end

data/lib/zorki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zorki
-  VERSION = "0.1.24"
+  VERSION = "0.1.27"
 end

data/lib/zorki.rb

@@ -28,7 +28,7 @@ module Zorki
     end
 
     def to_honeybadger_context
-      additional_data
+      @additional_data
     end
   end
 
@@ -39,7 +39,7 @@ module Zorki
     end
 
     def to_honeybadger_context
-      additional_data
+      @additional_data
     end
   end
 

data/zorki.gemspec

@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "apparition" # A Chrome driver for Capybara
   spec.add_dependency "typhoeus" # For making API requests
   spec.add_dependency "oj" # A faster JSON parser/loader than stdlib
-  spec.add_dependency "selenium-webdriver", "~> 4.19" # Webdriver selenium
+  spec.add_dependency "selenium-webdriver", "~> 4.24.0" # Webdriver selenium
   spec.add_dependency "selenium-devtools" # Allow us to intercept requests
 
   spec.add_development_dependency "debug"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zorki
 version: !ruby/object:Gem::Version
-  version: 0.1.24
+  version: 0.1.27
 platform: ruby
 authors:
 - Christopher Guess
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-06-19 00:00:00.000000000 Z
+date: 2024-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.19'
+        version: 4.24.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.19'
+        version: 4.24.0
 - !ruby/object:Gem::Dependency
   name: selenium-devtools
   requirement: !ruby/object:Gem::Requirement