zorki 0.1.9 → 0.1.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1eadd7ea03af064623b2e0a3deebc32bfa424ac60aab4713b61e3db94c10ee66
-  data.tar.gz: c4cd5d7bb8148eb23fb7d5b80f08f9118a9b39f717ca395a38b1abca34a7ce70
+  metadata.gz: '0228694f5aafe38a9856efe3a4f9f042b94aeecbd5cb7ec919326f48c0b8000e'
+  data.tar.gz: c54da1aa5b98d3166a047a24022b74356538e7dcb974e032bc5236065c5edd4c
 SHA512:
-  metadata.gz: dcfc9a4129cf62063c34461cfb1ee119ee702a64ed892b10909ddacb4cae544e7875eed7a53355db60c67f590292be951cd515d516b75456aae281a29ea3cbc5
-  data.tar.gz: feb1ca4d372217487c02311241ae9c4379eacc82336cc91e48c7f80ba68bef3538a14b0f17fb77e254d68203c77d4dcba9559b832c6dd9eda4879426a0e19c83
+  metadata.gz: 30b2953e778ce0a6e36f221350c6760974de7279c764111cfe2327f3bcc5f11a93d19a4747d20c35260038e07c06bad2c3d2fc704b4878c23a41ba309ae689d9
+  data.tar.gz: 99ed979bcc6f4f0758bca49d34580333ba42c35d74ce3ccfe52871acb9665d3841e0d54ed68bb3af2cf1ef366ebb1f3f9ea9136784958778751d7142f09e5b38

data/.gitignore

@@ -10,3 +10,5 @@
 .byebug_history
 
 selenium-server-*
+
+zorki-*.gem

data/Gemfile.lock

@@ -1,33 +1,18 @@
 PATH
   remote: .
   specs:
-    zorki (0.1.8)
+    zorki (0.1.9)
       apparition
       capybara
       oj
       selenium-devtools
-      selenium-webdriver
+      selenium-webdriver (~> 4.19)
       typhoeus
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (7.1.0.beta1)
-      actionview (= 7.1.0.beta1)
-      activesupport (= 7.1.0.beta1)
-      nokogiri (>= 1.8.5)
-      rack (>= 2.2.4)
-      rack-session (>= 1.0.1)
-      rack-test (>= 0.6.3)
-      rails-dom-testing (~> 2.2)
-      rails-html-sanitizer (~> 1.6)
-    actionview (7.1.0.beta1)
-      activesupport (= 7.1.0.beta1)
-      builder (~> 3.1)
-      erubi (~> 1.11)
-      rails-dom-testing (~> 2.2)
-      rails-html-sanitizer (~> 1.6)
-    activesupport (7.1.0.beta1)
+    activesupport (7.1.3.2)
       base64
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -37,125 +22,98 @@ GEM
       minitest (>= 5.1)
       mutex_m
       tzinfo (~> 2.0)
-    addressable (2.8.5)
+    addressable (2.8.6)
       public_suffix (>= 2.0.2, < 6.0)
     apparition (0.6.0)
       capybara (~> 3.13, < 4)
       websocket-driver (>= 0.6.5)
     ast (2.4.2)
-    base64 (0.1.1)
-    bigdecimal (3.1.4)
-    builder (3.2.4)
-    capybara (3.39.2)
+    base64 (0.2.0)
+    bigdecimal (3.1.7)
+    capybara (3.40.0)
       addressable
       matrix
       mini_mime (>= 0.1.3)
-      nokogiri (~> 1.8)
+      nokogiri (~> 1.11)
       rack (>= 1.6.0)
       rack-test (>= 0.6.3)
       regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
-    crass (1.0.6)
     curb (1.0.5)
-    debug (1.8.0)
-      irb (>= 1.5.0)
-      reline (>= 0.3.1)
+    debug (1.9.1)
+      irb (~> 1.10)
+      reline (>= 0.3.8)
     dotenv (2.7.6)
-    drb (2.1.1)
-      ruby2_keywords
-    erubi (1.12.0)
+    drb (2.2.1)
     ethon (0.16.0)
       ffi (>= 1.15.0)
-    ffi (1.15.5)
-    i18n (1.14.1)
+    ffi (1.16.3)
+    i18n (1.14.4)
       concurrent-ruby (~> 1.0)
-    io-console (0.6.0)
-    irb (1.8.1)
+    io-console (0.7.2)
+    irb (1.12.0)
       rdoc
-      reline (>= 0.3.8)
-    json (2.6.3)
+      reline (>= 0.4.2)
+    json (2.7.1)
     language_server-protocol (3.17.0.3)
-    loofah (2.21.3)
-      crass (~> 1.0.2)
-      nokogiri (>= 1.12.0)
     matrix (0.4.2)
     mini_mime (1.1.5)
-    minitest (5.20.0)
-    mutex_m (0.1.2)
-    nokogiri (1.15.4-arm64-darwin)
+    minitest (5.22.3)
+    mutex_m (0.2.0)
+    nokogiri (1.16.3-arm64-darwin)
       racc (~> 1.4)
-    oj (3.16.1)
-    parallel (1.23.0)
-    parser (3.2.2.3)
+    oj (3.16.3)
+      bigdecimal (>= 3.0)
+    parallel (1.24.0)
+    parser (3.3.0.5)
       ast (~> 2.4.1)
       racc
-    psych (5.1.0)
+    psych (5.1.2)
       stringio
-    public_suffix (5.0.3)
-    racc (1.7.1)
+    public_suffix (5.0.4)
+    racc (1.7.3)
     rack (3.0.8)
-    rack-session (2.0.0)
-      rack (>= 3.0.0)
     rack-test (2.1.0)
       rack (>= 1.3)
-    rackup (2.1.0)
-      rack (>= 3)
-      webrick (~> 1.8)
-    rails-dom-testing (2.2.0)
-      activesupport (>= 5.0.0)
-      minitest
-      nokogiri (>= 1.6)
-    rails-html-sanitizer (1.6.0)
-      loofah (~> 2.21)
-      nokogiri (~> 1.14)
-    railties (7.1.0.beta1)
-      actionpack (= 7.1.0.beta1)
-      activesupport (= 7.1.0.beta1)
-      irb
-      rackup (>= 1.0.0)
-      rake (>= 12.2)
-      thor (~> 1.0, >= 1.2.2)
-      zeitwerk (~> 2.6)
     rainbow (3.1.1)
-    rake (13.0.6)
-    rdoc (6.5.0)
+    rake (13.1.0)
+    rdoc (6.6.3.1)
       psych (>= 4.0.0)
-    regexp_parser (2.8.1)
-    reline (0.3.8)
+    regexp_parser (2.9.0)
+    reline (0.5.0)
       io-console (~> 0.5)
     rexml (3.2.6)
-    rubocop (1.56.3)
-      base64 (~> 0.1.1)
+    rubocop (1.62.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.2.3)
+      parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.28.1, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.29.0)
-      parser (>= 3.2.1.0)
-    rubocop-md (1.2.0)
+    rubocop-ast (1.31.2)
+      parser (>= 3.3.0.4)
+    rubocop-md (1.2.2)
       rubocop (>= 1.0)
-    rubocop-minitest (0.31.1)
-      rubocop (>= 1.39, < 2.0)
+    rubocop-minitest (0.35.0)
+      rubocop (>= 1.61, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
     rubocop-packaging (0.5.2)
       rubocop (>= 1.33, < 2.0)
-    rubocop-performance (1.19.0)
-      rubocop (>= 1.7.0, < 2.0)
-      rubocop-ast (>= 0.4.0)
+    rubocop-performance (1.20.2)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
     rubocop-rails (2.19.1)
       activesupport (>= 4.2.0)
       rack (>= 1.1)
       rubocop (>= 1.33.0, < 2.0)
-    rubocop-rails_config (1.14.1)
-      railties (>= 5.0)
-      rubocop (>= 1.48.0)
+    rubocop-rails_config (1.16.0)
+      rubocop (>= 1.57.0)
       rubocop-ast (>= 1.26.0)
       rubocop-md
       rubocop-minitest (~> 0.22)
@@ -163,29 +121,26 @@ GEM
       rubocop-performance (~> 1.11)
       rubocop-rails (~> 2.0)
     ruby-progressbar (1.13.0)
-    ruby2_keywords (0.0.5)
     rubyzip (2.3.2)
-    selenium-devtools (0.120.0)
+    selenium-devtools (0.123.0)
       selenium-webdriver (~> 4.2)
-    selenium-webdriver (4.16.0)
+    selenium-webdriver (4.19.0)
+      base64 (~> 0.2)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
-    stringio (3.0.8)
-    thor (1.2.2)
-    typhoeus (1.4.0)
+    stringio (3.1.0)
+    typhoeus (1.4.1)
       ethon (>= 0.9.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.4.2)
-    webrick (1.8.1)
+    unicode-display_width (2.5.0)
     websocket (1.2.10)
     websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.6.11)
 
 PLATFORMS
   arm64-darwin-22

data/lib/zorki/scrapers/post_scraper.rb

@@ -5,6 +5,28 @@ require "typhoeus"
 module Zorki
   class PostScraper < Scraper
     def parse(id)
+      count = 0
+
+      until count == 2
+        puts "Retrieving ID #{id}"
+
+        begin
+          result = attempt_parse(id)
+          break
+        rescue ImageRequestZeroSize
+          debugger
+          # If the image is zero size, we retry
+          puts "Zero sized image found, retrying #{count}"
+          count += 1
+        end
+      end
+
+      raise ImageRequestZeroSize if count == 5
+
+      result
+    end
+
+    def attempt_parse(id)
       # Stuff we need to get from the DOM (implemented is starred):
       # - User *
       # - Text *

data/lib/zorki/scrapers/scraper.rb

@@ -171,6 +171,8 @@ module Zorki
     end
 
     def login
+      puts "Attempting to login..."
+
       # Reset the sessions so that there's nothing laying around
       # page.driver.browser.close
 
@@ -184,7 +186,10 @@ module Zorki
 
       # We don't have to login if we already are
       begin
-        return if find_field("Search", wait: 10).present?
+        if find_field("Search", wait: 10).present?
+          puts "Already logged in"
+          return
+        end
       rescue Capybara::ElementNotFound; end
 
       # Check if we're redirected to a login page, if we aren't we're already logged in
@@ -193,24 +198,31 @@ module Zorki
       # Try to log in
       loop_count = 0
       while loop_count < 5 do
+        puts "Attempting to fill login field ##{loop_count}"
+
         fill_in("username", with: ENV["INSTAGRAM_USER_NAME"])
         fill_in("password", with: ENV["INSTAGRAM_PASSWORD"])
 
         begin
-          click_button("Log in", exact_text: true) # Note: "Log in" (lowercase `in`) instead redirects to Facebook's login page
+          find_button("Log in").click() # Note: "Log in" (lowercase `in`) should be exact instead, it redirects to Facebook's login page
         rescue Capybara::ElementNotFound; end # If we can't find it don't break horribly, just keep waiting
 
-        break unless has_css?('p[data-testid="login-error-message"', wait: 10)
+        break unless has_css?('p[data-testid="login-error-message"', wait: 3)
         loop_count += 1
-        sleep(rand * 10.3)
+        random_length = rand(1...2)
+        puts "Sleeping for #{random_length} seconds"
+        sleep(random_length)
       end
 
       # Sometimes Instagram just... doesn't let you log in
       raise "Instagram not accessible" if loop_count == 5
 
+      puts "Login successful"
       # No we don't want to save our login credentials
       begin
-        click_on("Save Info")
+        puts "Checking and clearing Save Info button"
+
+        find_button("Save Info").click()
       rescue Capybara::ElementNotFound; end
     end
 

data/lib/zorki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zorki
-  VERSION = "0.1.9"
+  VERSION = "0.1.21"
 end

data/lib/zorki.rb

@@ -46,11 +46,34 @@ module Zorki
     end
   end
 
+  class ImageRequestZeroSize < RetryableError; end
+
   define_setting :temp_storage_location, "tmp/zorki"
 
   # Get an image from a URL and save to a temp folder set in the configuration under
   # temp_storage_location
+
+  # We do this because sometimes the images are coming back sized zero
   def self.retrieve_media(url)
+    count = 0
+
+    until count == 5
+      temp_file_name = attempt_retrieve_media(url)
+
+      # If it's more than 1kb return properly
+      return temp_file_name if File.size(temp_file_name) > 100
+
+      # Delete the file since we want to retry
+      debugger
+
+      File.delete(temp_file_name)
+      count += 1
+    end
+
+    raise(ImageRequestZeroSize)
+  end
+
+  def self.attempt_retrieve_media(url)
     response = Typhoeus.get(url)
 
     # Get the file extension if it's in the file
@@ -69,6 +92,7 @@ module Zorki
     # We do this in case the folder isn't created yet, since it's a temp folder we'll just do so
     self.create_temp_storage_location
     File.binwrite(temp_file_name, response.body)
+
     temp_file_name
   end
 

data/zorki.gemspec

@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "apparition" # A Chrome driver for Capybara
   spec.add_dependency "typhoeus" # For making API requests
   spec.add_dependency "oj" # A faster JSON parser/loader than stdlib
-  spec.add_dependency "selenium-webdriver" # Webdriver selenium
+  spec.add_dependency "selenium-webdriver", "~> 4.19" # Webdriver selenium
   spec.add_dependency "selenium-devtools" # Allow us to intercept requests
 
   spec.add_development_dependency "debug"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zorki
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.21
 platform: ruby
 authors:
 - Christopher Guess
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-03-27 00:00:00.000000000 Z
+date: 2024-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -70,16 +70,16 @@ dependencies:
   name: selenium-webdriver
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.19'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.19'
 - !ruby/object:Gem::Dependency
   name: selenium-devtools
   requirement: !ruby/object:Gem::Requirement