zorki 0.1.7 → 0.1.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 469ec1ebba03ba1d24f78b28bbd6f1b939b94db49b1e61e6e47647e129ed64c1
-  data.tar.gz: 96655c2182ac16467e581be7965f5588d59f4ea0c181085d5a5c1f7575882b02
+  metadata.gz: 1e635e294bb6e73e0e57481ba79d23c603bab3ebbd73bbb4939d12f615644098
+  data.tar.gz: 876ce40814980d04a962c8002662bf7462a269fa7b6885d89c419b1a2a64c8e0
 SHA512:
-  metadata.gz: ec36d4c67c01d9a1198d3916b28f5b4fbc87f4f084ee732a15d268027d6ad164540ea87686ecd0548145d0a85cb3d0028f74e6ddd1d228d291cae78609b655f4
-  data.tar.gz: cf1662ba6bef2b0d3cf5eeb2d879efb5f69efa265268ccf4b6b6156b1b6f440a29a4cbaf2c7dd3f76a4b25a3c1f89d1b9fd384c953cea345169ed0061afbfd54
+  metadata.gz: 552de3b78d4bce7c18d6e0df96d3fdb4eb2b1b43e35961a64617e738510be4979453fe496a55e642329f47d2e4fc8ced52ff390263f39c010693eee4e71a9074
+  data.tar.gz: 07be9124885ef2eedf0b7433385e1a29a1c1fd33c58686530d99b7c9767784df6b1b7343c57c97157fed756c196d33125b35e621da9a7bf4737da1855a05c911

data/.gitignore

@@ -10,3 +10,5 @@
 .byebug_history
 
 selenium-server-*
+
+zorki-*.gem

data/Gemfile.lock

@@ -1,33 +1,18 @@
 PATH
   remote: .
   specs:
-    zorki (0.1.6)
+    zorki (0.1.9)
       apparition
       capybara
       oj
       selenium-devtools
-      selenium-webdriver
+      selenium-webdriver (~> 4.19)
       typhoeus
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (7.1.0.beta1)
-      actionview (= 7.1.0.beta1)
-      activesupport (= 7.1.0.beta1)
-      nokogiri (>= 1.8.5)
-      rack (>= 2.2.4)
-      rack-session (>= 1.0.1)
-      rack-test (>= 0.6.3)
-      rails-dom-testing (~> 2.2)
-      rails-html-sanitizer (~> 1.6)
-    actionview (7.1.0.beta1)
-      activesupport (= 7.1.0.beta1)
-      builder (~> 3.1)
-      erubi (~> 1.11)
-      rails-dom-testing (~> 2.2)
-      rails-html-sanitizer (~> 1.6)
-    activesupport (7.1.0.beta1)
+    activesupport (7.1.3.2)
       base64
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -37,122 +22,98 @@ GEM
       minitest (>= 5.1)
       mutex_m
       tzinfo (~> 2.0)
-    addressable (2.8.5)
+    addressable (2.8.6)
       public_suffix (>= 2.0.2, < 6.0)
     apparition (0.6.0)
       capybara (~> 3.13, < 4)
       websocket-driver (>= 0.6.5)
     ast (2.4.2)
-    base64 (0.1.1)
-    bigdecimal (3.1.4)
-    builder (3.2.4)
-    capybara (3.39.2)
+    base64 (0.2.0)
+    bigdecimal (3.1.7)
+    capybara (3.40.0)
       addressable
       matrix
       mini_mime (>= 0.1.3)
-      nokogiri (~> 1.8)
+      nokogiri (~> 1.11)
       rack (>= 1.6.0)
       rack-test (>= 0.6.3)
       regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
-    crass (1.0.6)
     curb (1.0.5)
+    debug (1.9.1)
+      irb (~> 1.10)
+      reline (>= 0.3.8)
     dotenv (2.7.6)
-    drb (2.1.1)
-      ruby2_keywords
-    erubi (1.12.0)
+    drb (2.2.1)
     ethon (0.16.0)
       ffi (>= 1.15.0)
-    ffi (1.15.5)
-    i18n (1.14.1)
+    ffi (1.16.3)
+    i18n (1.14.4)
       concurrent-ruby (~> 1.0)
-    io-console (0.6.0)
-    irb (1.8.1)
+    io-console (0.7.2)
+    irb (1.12.0)
       rdoc
-      reline (>= 0.3.8)
-    json (2.6.3)
+      reline (>= 0.4.2)
+    json (2.7.1)
     language_server-protocol (3.17.0.3)
-    loofah (2.21.3)
-      crass (~> 1.0.2)
-      nokogiri (>= 1.12.0)
     matrix (0.4.2)
     mini_mime (1.1.5)
-    minitest (5.20.0)
-    mutex_m (0.1.2)
-    nokogiri (1.15.4-arm64-darwin)
+    minitest (5.22.3)
+    mutex_m (0.2.0)
+    nokogiri (1.16.3-arm64-darwin)
       racc (~> 1.4)
-    oj (3.16.1)
-    parallel (1.23.0)
-    parser (3.2.2.3)
+    oj (3.16.3)
+      bigdecimal (>= 3.0)
+    parallel (1.24.0)
+    parser (3.3.0.5)
       ast (~> 2.4.1)
       racc
-    psych (5.1.0)
+    psych (5.1.2)
       stringio
-    public_suffix (5.0.3)
-    racc (1.7.1)
+    public_suffix (5.0.4)
+    racc (1.7.3)
     rack (3.0.8)
-    rack-session (2.0.0)
-      rack (>= 3.0.0)
     rack-test (2.1.0)
       rack (>= 1.3)
-    rackup (2.1.0)
-      rack (>= 3)
-      webrick (~> 1.8)
-    rails-dom-testing (2.2.0)
-      activesupport (>= 5.0.0)
-      minitest
-      nokogiri (>= 1.6)
-    rails-html-sanitizer (1.6.0)
-      loofah (~> 2.21)
-      nokogiri (~> 1.14)
-    railties (7.1.0.beta1)
-      actionpack (= 7.1.0.beta1)
-      activesupport (= 7.1.0.beta1)
-      irb
-      rackup (>= 1.0.0)
-      rake (>= 12.2)
-      thor (~> 1.0, >= 1.2.2)
-      zeitwerk (~> 2.6)
     rainbow (3.1.1)
-    rake (13.0.6)
-    rdoc (6.5.0)
+    rake (13.1.0)
+    rdoc (6.6.3.1)
       psych (>= 4.0.0)
-    regexp_parser (2.8.1)
-    reline (0.3.8)
+    regexp_parser (2.9.0)
+    reline (0.5.0)
       io-console (~> 0.5)
     rexml (3.2.6)
-    rubocop (1.56.3)
-      base64 (~> 0.1.1)
+    rubocop (1.62.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.2.3)
+      parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.28.1, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.29.0)
-      parser (>= 3.2.1.0)
-    rubocop-md (1.2.0)
+    rubocop-ast (1.31.2)
+      parser (>= 3.3.0.4)
+    rubocop-md (1.2.2)
       rubocop (>= 1.0)
-    rubocop-minitest (0.31.1)
-      rubocop (>= 1.39, < 2.0)
+    rubocop-minitest (0.35.0)
+      rubocop (>= 1.61, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
     rubocop-packaging (0.5.2)
       rubocop (>= 1.33, < 2.0)
-    rubocop-performance (1.19.0)
-      rubocop (>= 1.7.0, < 2.0)
-      rubocop-ast (>= 0.4.0)
+    rubocop-performance (1.20.2)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
     rubocop-rails (2.19.1)
       activesupport (>= 4.2.0)
       rack (>= 1.1)
       rubocop (>= 1.33.0, < 2.0)
-    rubocop-rails_config (1.14.1)
-      railties (>= 5.0)
-      rubocop (>= 1.48.0)
+    rubocop-rails_config (1.16.0)
+      rubocop (>= 1.57.0)
       rubocop-ast (>= 1.26.0)
       rubocop-md
       rubocop-minitest (~> 0.22)
@@ -160,35 +121,33 @@ GEM
       rubocop-performance (~> 1.11)
       rubocop-rails (~> 2.0)
     ruby-progressbar (1.13.0)
-    ruby2_keywords (0.0.5)
     rubyzip (2.3.2)
-    selenium-devtools (0.119.0)
+    selenium-devtools (0.123.0)
       selenium-webdriver (~> 4.2)
-    selenium-webdriver (4.15.0)
+    selenium-webdriver (4.19.0)
+      base64 (~> 0.2)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
-    stringio (3.0.8)
-    thor (1.2.2)
-    typhoeus (1.4.0)
+    stringio (3.1.0)
+    typhoeus (1.4.1)
       ethon (>= 0.9.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.4.2)
-    webrick (1.8.1)
+    unicode-display_width (2.5.0)
     websocket (1.2.10)
     websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.6.11)
 
 PLATFORMS
   arm64-darwin-22
 
 DEPENDENCIES
   curb (~> 1.0, >= 1.0.5)
+  debug
   dotenv (~> 2.7.6)
   minitest (~> 5.0)
   rack (= 3.0.8)

data/lib/zorki/scrapers/post_scraper.rb

@@ -97,7 +97,6 @@ module Zorki
 
       # This has to run last since it switches pages
       user = User.lookup([username]).first
-      page.quit
 
       {
         images: images,

data/lib/zorki/scrapers/scraper.rb

@@ -114,27 +114,32 @@ module Zorki
         #   e.attributes.has_key?("type") && e.attributes["type"].value == "application/ld+json"
         # end
 
-        elements = doc.search("script").map do |element|
-            element_json = nil
+        elements = doc.search("script").filter_map do |element|
+          parsed_element_json = nil
           begin
-            element_json = JSON.parse(element)
+            element_json = JSON.parse(element.text)
 
-            element_json = element_json["require"].first.last.first["__bbox"]["require"].first.last.last["__bbox"]["result"]["data"]["xdt_api__v1__media__shortcode__web_info"]
-          rescue StandardError => e
+            parsed_element_json = element_json["require"].first.last.first["__bbox"]["require"].first.last.last["__bbox"]["result"]["data"]["xdt_api__v1__media__shortcode__web_info"]
+          rescue StandardError
             next
           end
 
-          element_json
-        end.compact
+          parsed_element_json
+        end
 
         if elements&.empty?
-          raise ContentUnavailableError
+          raise ContentUnavailableError.new("Cannot find anything", additional_data: { page_source: page.driver.browser.page_source, elements: elements })
         end
+
         return elements
       end
 
-      raise ContentUnavailableError if response_body.nil?
+      raise ContentUnavailableError.new("Response body nil") if response_body.nil?
       Oj.load(response_body)
+    ensure
+      page.quit
+      # TRY THIS TO MAKE SURE CHROME GETS CLOSED?
+      # We may also want to not do this and make sure the same browser is reused instead for cookie purposes
     end
 
   private
@@ -166,8 +171,9 @@ module Zorki
     end
 
     def login
+      puts "Attempting to login..."
       # Reset the sessions so that there's nothing laying around
-      page.quit
+      # page.driver.browser.close
 
       # Check if we're on a Instagram page already, if not visit it.
       unless page.driver.browser.current_url.include? "instagram.com"
@@ -188,16 +194,20 @@ module Zorki
       # Try to log in
       loop_count = 0
       while loop_count < 5 do
+        puts "Attempting to fill login field ##{loop_count}"
+
         fill_in("username", with: ENV["INSTAGRAM_USER_NAME"])
         fill_in("password", with: ENV["INSTAGRAM_PASSWORD"])
 
         begin
-          click_button("Log in", exact_text: true) # Note: "Log in" (lowercase `in`) instead redirects to Facebook's login page
+          find_button("Log in").click() # Note: "Log in" (lowercase `in`) should be exact instead, it redirects to Facebook's login page
         rescue Capybara::ElementNotFound; end # If we can't find it don't break horribly, just keep waiting
 
-        break unless has_css?('p[data-testid="login-error-message"', wait: 10)
+        break unless has_css?('p[data-testid="login-error-message"', wait: 3)
         loop_count += 1
-        sleep(rand * 10.3)
+        random_length = rand(1...2)
+        puts "Sleeping for #{random_length} seconds"
+        sleep(random_length)
       end
 
       # Sometimes Instagram just... doesn't let you log in
@@ -205,7 +215,9 @@ module Zorki
 
       # No we don't want to save our login credentials
       begin
-        click_on("Save Info")
+        puts "Checking and clearing Save Info button..."
+
+        find_button("Save Info").click()
       rescue Capybara::ElementNotFound; end
     end
 
@@ -242,6 +254,15 @@ module Zorki
       # Multiply everything and insure we get an integer back
       (number * multiplier).to_i
     end
+
+    # def reset_window
+    #   old_handle = page.driver.browser.window_handle
+    #   page.driver.browser.switch_to.new_window(:window)
+    #   new_handle = page.driver.browser.window_handle
+    #   page.driver.browser.switch_to.window(old_handle)
+    #   page.driver.browser.close
+    #   page.driver.browser.switch_to.window(new_handle)
+    # end
   end
 end
 

data/lib/zorki/scrapers/user_scraper.rb

@@ -21,6 +21,10 @@ module Zorki
       graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "?username=")
       graphql_script = graphql_script.first if graphql_script.class == Array
 
+      if graphql_script.nil?
+        graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "web_profile_info")
+      end
+
       if graphql_script.has_key?("author") && !graphql_script["author"].nil?
         user = graphql_script["author"]
 

data/lib/zorki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zorki
-  VERSION = "0.1.7"
+  VERSION = "0.1.20"
 end

data/lib/zorki.rb

@@ -20,8 +20,15 @@ module Zorki
   end
 
   class ContentUnavailableError < Error
-    def initialize(msg = "Zorki could not find content requested")
-      super
+    attr_reader :additional_data
+
+    def initialize(msg = "Zorki could not find content requested", additional_data: {})
+      super(msg)
+      @additional_data = additional_data
+    end
+
+    def to_honeybadger_context
+      additional_data
     end
   end
 

data/zorki.gemspec

@@ -29,15 +29,15 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  # Uncomment to register a new dependency of your gem
-  # spec.add_dependency "example-gem", "~> 1.0"
   spec.add_dependency "capybara" # For scraping and running browsers
   spec.add_dependency "apparition" # A Chrome driver for Capybara
   spec.add_dependency "typhoeus" # For making API requests
   spec.add_dependency "oj" # A faster JSON parser/loader than stdlib
-  spec.add_dependency "selenium-webdriver" # Webdriver selenium
+  spec.add_dependency "selenium-webdriver", "~> 4.19" # Webdriver selenium
   spec.add_dependency "selenium-devtools" # Allow us to intercept requests
 
+  spec.add_development_dependency "debug"
+
   # For more information and examples about making a new gem, checkout our
   # guide at: https://bundler.io/guides/creating_gem.html
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zorki
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.20
 platform: ruby
 authors:
 - Christopher Guess
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-12-05 00:00:00.000000000 Z
+date: 2024-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -68,6 +68,20 @@ dependencies:
         version: '0'
 - !ruby/object:Gem::Dependency
   name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.19'
+- !ruby/object:Gem::Dependency
+  name: selenium-devtools
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -81,13 +95,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: selenium-devtools
+  name: debug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements: