zorki 0.1.24 → 0.1.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa95bb5ca131ca6b7faed2aab200329579b77012cb079da75a0ad90a60daa5bd
-  data.tar.gz: 82eb0cc29af2cfaeafa8774e027904959f953a50faa03d71dbcbeba9595ac520
+  metadata.gz: ec1dba1c80d66f33d1e50fc104511a5842091529c6d2b1e3f4758508ae37468a
+  data.tar.gz: b510d48cbc12d0e9b70fb17fb91aeffdad51d85f85e5813695df1c86279894db
 SHA512:
-  metadata.gz: 127638ce83ed09be71f194ba3e5dc269374890a1bd89874c8f158896a7664d41724698a258916f63e373b1c6f3dd3b4d315fac49a15e17ff457035ec4d345120
-  data.tar.gz: 105b7e148774a82640ed48bffe46b21d941ce0a18987f59ef0e79a3a058077fbd9dda651811bbc421ff9fcc8323de152b2ac5afbff0e5b9d4e5c65d27f5ac85c
+  metadata.gz: 6463fc59b818e21d4c515212d04549b7a3b925433d7e9df79ff376a786f74e028ab99146b8422c64384470443d88835854717e3a19a7c68bb3a767e7277f1c39
+  data.tar.gz: 173b570bb5eb62d759a488c21aff0092c633db1f0f1a4fa771e46ad32141a2016386f698e724cbc1a7dced5ad66a9d7b22add8f7dd533d8dc88ec009ca9a2814

data/Gemfile.lock

@@ -1,35 +1,36 @@
 PATH
   remote: .
   specs:
-    zorki (0.1.9)
+    zorki (0.1.25)
       apparition
       capybara
       oj
       selenium-devtools
-      selenium-webdriver (~> 4.19)
+      selenium-webdriver (~> 4.24.0)
       typhoeus
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.1.3.2)
+    activesupport (7.2.1)
       base64
       bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+      concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
       drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      mutex_m
-      tzinfo (~> 2.0)
-    addressable (2.8.6)
-      public_suffix (>= 2.0.2, < 6.0)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+    addressable (2.8.7)
+      public_suffix (>= 2.0.2, < 7.0)
     apparition (0.6.0)
       capybara (~> 3.13, < 4)
       websocket-driver (>= 0.6.5)
     ast (2.4.2)
     base64 (0.2.0)
-    bigdecimal (3.1.7)
+    bigdecimal (3.1.8)
     capybara (3.40.0)
       addressable
       matrix
@@ -39,75 +40,76 @@ GEM
       rack-test (>= 0.6.3)
       regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
-    concurrent-ruby (1.2.3)
+    concurrent-ruby (1.3.4)
     connection_pool (2.4.1)
-    curb (1.0.5)
-    debug (1.9.1)
+    curb (1.0.6)
+    debug (1.9.2)
       irb (~> 1.10)
       reline (>= 0.3.8)
     dotenv (2.7.6)
     drb (2.2.1)
     ethon (0.16.0)
       ffi (>= 1.15.0)
-    ffi (1.16.3)
-    i18n (1.14.4)
+    ffi (1.17.0-arm64-darwin)
+    i18n (1.14.5)
       concurrent-ruby (~> 1.0)
     io-console (0.7.2)
-    irb (1.12.0)
-      rdoc
+    irb (1.14.0)
+      rdoc (>= 4.0.0)
       reline (>= 0.4.2)
-    json (2.7.1)
+    json (2.7.2)
     language_server-protocol (3.17.0.3)
+    logger (1.6.1)
     matrix (0.4.2)
     mini_mime (1.1.5)
-    minitest (5.22.3)
-    mutex_m (0.2.0)
-    nokogiri (1.16.3-arm64-darwin)
+    minitest (5.25.1)
+    nokogiri (1.16.7-arm64-darwin)
       racc (~> 1.4)
-    oj (3.16.3)
+    oj (3.16.6)
       bigdecimal (>= 3.0)
-    parallel (1.24.0)
-    parser (3.3.0.5)
+      ostruct (>= 0.2)
+    ostruct (0.6.0)
+    parallel (1.26.3)
+    parser (3.3.5.0)
       ast (~> 2.4.1)
       racc
     psych (5.1.2)
       stringio
-    public_suffix (5.0.4)
-    racc (1.7.3)
+    public_suffix (6.0.1)
+    racc (1.8.1)
     rack (3.0.8)
     rack-test (2.1.0)
       rack (>= 1.3)
     rainbow (3.1.1)
-    rake (13.1.0)
-    rdoc (6.6.3.1)
+    rake (13.2.1)
+    rdoc (6.7.0)
       psych (>= 4.0.0)
-    regexp_parser (2.9.0)
-    reline (0.5.0)
+    regexp_parser (2.9.2)
+    reline (0.5.10)
       io-console (~> 0.5)
-    rexml (3.2.6)
-    rubocop (1.62.1)
+    rexml (3.3.7)
+    rubocop (1.66.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.31.1, < 2.0)
+      regexp_parser (>= 2.4, < 3.0)
+      rubocop-ast (>= 1.32.2, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.31.2)
-      parser (>= 3.3.0.4)
-    rubocop-md (1.2.2)
-      rubocop (>= 1.0)
-    rubocop-minitest (0.35.0)
+    rubocop-ast (1.32.3)
+      parser (>= 3.3.1.0)
+    rubocop-md (1.2.3)
+      rubocop (>= 1.45)
+    rubocop-minitest (0.36.0)
       rubocop (>= 1.61, < 2.0)
       rubocop-ast (>= 1.31.1, < 2.0)
     rubocop-packaging (0.5.2)
       rubocop (>= 1.33, < 2.0)
-    rubocop-performance (1.20.2)
+    rubocop-performance (1.21.1)
       rubocop (>= 1.48.1, < 2.0)
-      rubocop-ast (>= 1.30.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
     rubocop-rails (2.19.1)
       activesupport (>= 4.2.0)
       rack (>= 1.1)
@@ -122,20 +124,22 @@ GEM
       rubocop-rails (~> 2.0)
     ruby-progressbar (1.13.0)
     rubyzip (2.3.2)
-    selenium-devtools (0.123.0)
+    securerandom (0.3.1)
+    selenium-devtools (0.128.0)
       selenium-webdriver (~> 4.2)
-    selenium-webdriver (4.19.0)
+    selenium-webdriver (4.24.0)
       base64 (~> 0.2)
+      logger (~> 1.4)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
-    stringio (3.1.0)
+    stringio (3.1.1)
     typhoeus (1.4.1)
       ethon (>= 0.9.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.5.0)
-    websocket (1.2.10)
+    unicode-display_width (2.6.0)
+    websocket (1.2.11)
     websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)

data/lib/zorki/scrapers/scraper.rb

@@ -141,12 +141,14 @@ module Zorki
         end
 
         if elements&.empty?
+          # debugger
           raise ContentUnavailableError.new("Cannot find anything", additional_data: { page_source: page.driver.browser.page_source, elements: elements })
         end
 
         return elements
       end
 
+      # debugger if response_body.nil?
       raise ContentUnavailableError.new("Response body nil") if response_body.nil?
       Oj.load(response_body)
     ensure

data/lib/zorki/scrapers/user_scraper.rb

@@ -16,13 +16,30 @@ module Zorki
       #   - *description
       #   - *links
       # - *Profile image
-      login
 
-      graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "graphql", "data,user,full_name")
-      graphql_script = graphql_script.first if graphql_script.class == Array
+      graphql_script = nil
+      count = 0
+      loop do
+        raise Zorki::UserScrapingError.new("Zorki could not find user #{username}", additional_data: { username: username }) if count > 3
 
-      if graphql_script.nil?
-        graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "web_profile_info")
+        print "Scraping user #{username}... (attempt #{count + 1})\n"
+        begin
+          login
+
+          graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "graphql", "data,user,media_count")
+          graphql_script = graphql_script.first if graphql_script.class == Array
+
+          if graphql_script.nil?
+            graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "web_profile_info")
+          end
+        rescue Zorki::ContentUnavailableError => e
+          count += 1
+          page.driver.browser.navigate.to("https://www.instagram.com") # we want to go back to the main page so we start from scratch
+          sleep 10
+          next
+        end
+
+        break
       end
 
       if graphql_script.has_key?("author") && !graphql_script["author"].nil?
@@ -36,10 +53,14 @@ module Zorki
           ["https://schema.org/FilmAction", "http://schema.org/WriteAction"].include?(stat["interactionType"])
         end.first
 
+        # number_of_posts = graphql_script["data"]["user"]["media_count"] if number_of_posts.nil?
+
         number_of_followers = graphql_script["interactionStatistic"].select do |stat|
           stat["interactionType"] == "http://schema.org/FollowAction"
         end.first
 
+        # number_of_followers = graphql_script["data"]["user"]["follower_count"] if number_of_followers.nil?
+
         begin
           profile_image_url = user["image"]
           {
@@ -76,7 +97,8 @@ module Zorki
           profile_image_url: profile_image_url
         }
       end
-    rescue Zorki::ContentUnavailableError
+    rescue Zorki::ContentUnavailableError => e
+      debugger
       raise Zorki::UserScrapingError.new("Zorki could not find user #{username}", additional_data: { username: username })
     end
   end

data/lib/zorki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zorki
-  VERSION = "0.1.24"
+  VERSION = "0.1.26"
 end

data/lib/zorki.rb

@@ -28,7 +28,7 @@ module Zorki
     end
 
     def to_honeybadger_context
-      additional_data
+      @additional_data
     end
   end
 
@@ -39,7 +39,7 @@ module Zorki
     end
 
     def to_honeybadger_context
-      additional_data
+      @additional_data
     end
   end
 

data/zorki.gemspec

@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "apparition" # A Chrome driver for Capybara
   spec.add_dependency "typhoeus" # For making API requests
   spec.add_dependency "oj" # A faster JSON parser/loader than stdlib
-  spec.add_dependency "selenium-webdriver", "~> 4.19" # Webdriver selenium
+  spec.add_dependency "selenium-webdriver", "~> 4.24.0" # Webdriver selenium
   spec.add_dependency "selenium-devtools" # Allow us to intercept requests
 
   spec.add_development_dependency "debug"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zorki
 version: !ruby/object:Gem::Version
-  version: 0.1.24
+  version: 0.1.26
 platform: ruby
 authors:
 - Christopher Guess
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-06-19 00:00:00.000000000 Z
+date: 2024-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.19'
+        version: 4.24.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.19'
+        version: 4.24.0
 - !ruby/object:Gem::Dependency
   name: selenium-devtools
   requirement: !ruby/object:Gem::Requirement