zorki 0.1.23 → 0.1.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59b7b2ee84517a52b4c4bfaa300fe18d5981789c92c5983cc07d2cbe21323a79
-  data.tar.gz: 6d24c1955520c08164da77783dbbc17d8db09edfa8de30103a3ddfcdaeaf8064
+  metadata.gz: ec1dba1c80d66f33d1e50fc104511a5842091529c6d2b1e3f4758508ae37468a
+  data.tar.gz: b510d48cbc12d0e9b70fb17fb91aeffdad51d85f85e5813695df1c86279894db
 SHA512:
-  metadata.gz: 3bb4c7df50e497920f6e21216268a2640128f0b0fee6d443827ffe372f720a63b03fe6ad0c62d2a78862698989e7b62929bd9d0e9c54e5dcf12c8a5ccca85719
-  data.tar.gz: 1cee6eafb78aaf2bcadc855f44676501996ce83c9fc5a3ecafc5303fc7a62d2ee7aa0730b78477c8578fd1edc3e897317ee6014cfd35b38d5743816ebf8307e4
+  metadata.gz: 6463fc59b818e21d4c515212d04549b7a3b925433d7e9df79ff376a786f74e028ab99146b8422c64384470443d88835854717e3a19a7c68bb3a767e7277f1c39
+  data.tar.gz: 173b570bb5eb62d759a488c21aff0092c633db1f0f1a4fa771e46ad32141a2016386f698e724cbc1a7dced5ad66a9d7b22add8f7dd533d8dc88ec009ca9a2814

data/Gemfile.lock

@@ -1,35 +1,36 @@
 PATH
   remote: .
   specs:
-    zorki (0.1.9)
+    zorki (0.1.25)
       apparition
       capybara
       oj
       selenium-devtools
-      selenium-webdriver (~> 4.19)
+      selenium-webdriver (~> 4.24.0)
       typhoeus
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.1.3.2)
+    activesupport (7.2.1)
       base64
       bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+      concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
       drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      mutex_m
-      tzinfo (~> 2.0)
-    addressable (2.8.6)
-      public_suffix (>= 2.0.2, < 6.0)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+    addressable (2.8.7)
+      public_suffix (>= 2.0.2, < 7.0)
     apparition (0.6.0)
       capybara (~> 3.13, < 4)
       websocket-driver (>= 0.6.5)
     ast (2.4.2)
     base64 (0.2.0)
-    bigdecimal (3.1.7)
+    bigdecimal (3.1.8)
     capybara (3.40.0)
       addressable
       matrix
@@ -39,75 +40,76 @@ GEM
       rack-test (>= 0.6.3)
       regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
-    concurrent-ruby (1.2.3)
+    concurrent-ruby (1.3.4)
     connection_pool (2.4.1)
-    curb (1.0.5)
-    debug (1.9.1)
+    curb (1.0.6)
+    debug (1.9.2)
       irb (~> 1.10)
       reline (>= 0.3.8)
     dotenv (2.7.6)
     drb (2.2.1)
     ethon (0.16.0)
       ffi (>= 1.15.0)
-    ffi (1.16.3)
-    i18n (1.14.4)
+    ffi (1.17.0-arm64-darwin)
+    i18n (1.14.5)
       concurrent-ruby (~> 1.0)
     io-console (0.7.2)
-    irb (1.12.0)
-      rdoc
+    irb (1.14.0)
+      rdoc (>= 4.0.0)
       reline (>= 0.4.2)
-    json (2.7.1)
+    json (2.7.2)
     language_server-protocol (3.17.0.3)
+    logger (1.6.1)
     matrix (0.4.2)
     mini_mime (1.1.5)
-    minitest (5.22.3)
-    mutex_m (0.2.0)
-    nokogiri (1.16.3-arm64-darwin)
+    minitest (5.25.1)
+    nokogiri (1.16.7-arm64-darwin)
       racc (~> 1.4)
-    oj (3.16.3)
+    oj (3.16.6)
       bigdecimal (>= 3.0)
-    parallel (1.24.0)
-    parser (3.3.0.5)
+      ostruct (>= 0.2)
+    ostruct (0.6.0)
+    parallel (1.26.3)
+    parser (3.3.5.0)
       ast (~> 2.4.1)
       racc
     psych (5.1.2)
       stringio
-    public_suffix (5.0.4)
-    racc (1.7.3)
+    public_suffix (6.0.1)
+    racc (1.8.1)
     rack (3.0.8)
     rack-test (2.1.0)
       rack (>= 1.3)
     rainbow (3.1.1)
-    rake (13.1.0)
-    rdoc (6.6.3.1)
+    rake (13.2.1)
+    rdoc (6.7.0)
       psych (>= 4.0.0)
-    regexp_parser (2.9.0)
-    reline (0.5.0)
+    regexp_parser (2.9.2)
+    reline (0.5.10)
       io-console (~> 0.5)
-    rexml (3.2.6)
-    rubocop (1.62.1)
+    rexml (3.3.7)
+    rubocop (1.66.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.31.1, < 2.0)
+      regexp_parser (>= 2.4, < 3.0)
+      rubocop-ast (>= 1.32.2, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.31.2)
-      parser (>= 3.3.0.4)
-    rubocop-md (1.2.2)
-      rubocop (>= 1.0)
-    rubocop-minitest (0.35.0)
+    rubocop-ast (1.32.3)
+      parser (>= 3.3.1.0)
+    rubocop-md (1.2.3)
+      rubocop (>= 1.45)
+    rubocop-minitest (0.36.0)
       rubocop (>= 1.61, < 2.0)
       rubocop-ast (>= 1.31.1, < 2.0)
     rubocop-packaging (0.5.2)
       rubocop (>= 1.33, < 2.0)
-    rubocop-performance (1.20.2)
+    rubocop-performance (1.21.1)
       rubocop (>= 1.48.1, < 2.0)
-      rubocop-ast (>= 1.30.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
     rubocop-rails (2.19.1)
       activesupport (>= 4.2.0)
       rack (>= 1.1)
@@ -122,20 +124,22 @@ GEM
       rubocop-rails (~> 2.0)
     ruby-progressbar (1.13.0)
     rubyzip (2.3.2)
-    selenium-devtools (0.123.0)
+    securerandom (0.3.1)
+    selenium-devtools (0.128.0)
       selenium-webdriver (~> 4.2)
-    selenium-webdriver (4.19.0)
+    selenium-webdriver (4.24.0)
       base64 (~> 0.2)
+      logger (~> 1.4)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
-    stringio (3.1.0)
+    stringio (3.1.1)
     typhoeus (1.4.1)
       ethon (>= 0.9.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.5.0)
-    websocket (1.2.10)
+    unicode-display_width (2.6.0)
+    websocket (1.2.11)
     websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)

data/lib/zorki/scrapers/scraper.rb

@@ -78,6 +78,8 @@ module Zorki
             unless additional_search_parameters.nil?
               body_to_check = Oj.load(response.body)
 
+              debugger if body_to_check.include?("jokoy.komi.io")
+
               search_parameters = additional_search_parameters.split(",")
               search_parameters.each_with_index do |key, index|
                 break if body_to_check.nil?
@@ -120,11 +122,12 @@ module Zorki
           begin
             element_json = JSON.parse(element.text)
 
-            if element.text.include?("Dwayne")
+            if element.text.include?("jokoy.komi.io")
+              debugger
               # if element_json["require"].first.last.first["__bbox"].key?("require")
 
               #   element_json["require"].first.last.first["__bbox"]["require"].each do |x|
-              #     debugger if x.to_s.include?("Dwayne Johnson")
+              #     debugger if x.to_s.include?("Si mulut pelaut")
               #   end
               # end
             end
@@ -138,12 +141,14 @@ module Zorki
         end
 
         if elements&.empty?
+          # debugger
           raise ContentUnavailableError.new("Cannot find anything", additional_data: { page_source: page.driver.browser.page_source, elements: elements })
         end
 
         return elements
       end
 
+      # debugger if response_body.nil?
       raise ContentUnavailableError.new("Response body nil") if response_body.nil?
       Oj.load(response_body)
     ensure

data/lib/zorki/scrapers/user_scraper.rb

@@ -16,13 +16,30 @@ module Zorki
       #   - *description
       #   - *links
       # - *Profile image
-      login
 
-      graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "graphql", "data,user,full_name")
-      graphql_script = graphql_script.first if graphql_script.class == Array
+      graphql_script = nil
+      count = 0
+      loop do
+        raise Zorki::UserScrapingError.new("Zorki could not find user #{username}", additional_data: { username: username }) if count > 3
 
-      if graphql_script.nil?
-        graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "web_profile_info")
+        print "Scraping user #{username}... (attempt #{count + 1})\n"
+        begin
+          login
+
+          graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "graphql", "data,user,media_count")
+          graphql_script = graphql_script.first if graphql_script.class == Array
+
+          if graphql_script.nil?
+            graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "web_profile_info")
+          end
+        rescue Zorki::ContentUnavailableError => e
+          count += 1
+          page.driver.browser.navigate.to("https://www.instagram.com") # we want to go back to the main page so we start from scratch
+          sleep 10
+          next
+        end
+
+        break
       end
 
       if graphql_script.has_key?("author") && !graphql_script["author"].nil?
@@ -36,10 +53,14 @@ module Zorki
           ["https://schema.org/FilmAction", "http://schema.org/WriteAction"].include?(stat["interactionType"])
         end.first
 
+        # number_of_posts = graphql_script["data"]["user"]["media_count"] if number_of_posts.nil?
+
         number_of_followers = graphql_script["interactionStatistic"].select do |stat|
           stat["interactionType"] == "http://schema.org/FollowAction"
         end.first
 
+        # number_of_followers = graphql_script["data"]["user"]["follower_count"] if number_of_followers.nil?
+
         begin
           profile_image_url = user["image"]
           {
@@ -76,6 +97,9 @@ module Zorki
           profile_image_url: profile_image_url
         }
       end
+    rescue Zorki::ContentUnavailableError => e
+      debugger
+      raise Zorki::UserScrapingError.new("Zorki could not find user #{username}", additional_data: { username: username })
     end
   end
 end

data/lib/zorki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zorki
-  VERSION = "0.1.23"
+  VERSION = "0.1.26"
 end

data/lib/zorki.rb

@@ -28,7 +28,18 @@ module Zorki
     end
 
     def to_honeybadger_context
-      additional_data
+      @additional_data
+    end
+  end
+
+  class UserScrapingError < Error
+    def initialize(msg = "Zorki encountered an error scraping a user", additional_data: {})
+      super(msg)
+      @additional_data = additional_data
+    end
+
+    def to_honeybadger_context
+      @additional_data
     end
   end
 

data/zorki.gemspec

@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "apparition" # A Chrome driver for Capybara
   spec.add_dependency "typhoeus" # For making API requests
   spec.add_dependency "oj" # A faster JSON parser/loader than stdlib
-  spec.add_dependency "selenium-webdriver", "~> 4.19" # Webdriver selenium
+  spec.add_dependency "selenium-webdriver", "~> 4.24.0" # Webdriver selenium
   spec.add_dependency "selenium-devtools" # Allow us to intercept requests
 
   spec.add_development_dependency "debug"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zorki
 version: !ruby/object:Gem::Version
-  version: 0.1.23
+  version: 0.1.26
 platform: ruby
 authors:
 - Christopher Guess
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-05-29 00:00:00.000000000 Z
+date: 2024-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.19'
+        version: 4.24.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.19'
+        version: 4.24.0
 - !ruby/object:Gem::Dependency
   name: selenium-devtools
   requirement: !ruby/object:Gem::Requirement