zorki 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 384abb984483d4f38da44e77b7cd359e7d3aadea7c96765bc5b2837c3e79e99d
-  data.tar.gz: 48bf93b460b4c947cc9bf47f97dc469de124686cb763eccfa8ef7ef00e11f2b6
+  metadata.gz: 8e9aca1027df18f607fac0616a75a4b8ac11728cbac5de9205f014c57306f82e
+  data.tar.gz: c22375fd87090060642780ae2ec472d505f66c7ed1579d35d8eb0e9fe02bd976
 SHA512:
-  metadata.gz: e70b44d624a51df5a7ca6d9b6078cf3b44c8da1c07291aee60e4d554e2bc5705cfab3cea112ccbc79f6354d1092dca90330b7697f4e19606034857de361a08e3
-  data.tar.gz: b819d82e750f72cb55ab34b2a7b8332c70828220857501c22315480e644c2b88d5732f3b4074d7d240015a70dadace213b4a1e388145a6d50d544ef36b20b655
+  metadata.gz: b3e3117ca0903a23fd2303ec938d70d1d8dd8f82eeb510019832e25e286a4c5ae665c707cc62523dedf9617de9239072aa02d8877fb8e2fae2ea4f85ad14bbc9
+  data.tar.gz: 7a2b9d079041484f5553bf5be6fa391bbeb20c8660fb07df69cd7b6feb28d8f7d315e22353d89ff1ccedf7304e39dfbab9f8f18e9f25ad44cd7067e473d515d1

data/Gemfile

@@ -15,3 +15,5 @@ gem "rubocop-rails", "~> 2.19.1", require: false # Rails specific styles
 gem "rubocop-rails_config"
 
 gem "dotenv", "~> 2.7.6"
+
+gem "curb", "~> 1.0", ">= 1.0.5"

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    zorki (0.1.0)
+    zorki (0.1.2)
       apparition
       capybara
       oj
@@ -12,20 +12,20 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (7.0.4.3)
-      actionview (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
-      rack (~> 2.0, >= 2.2.0)
+    actionpack (7.0.5)
+      actionview (= 7.0.5)
+      activesupport (= 7.0.5)
+      rack (~> 2.0, >= 2.2.4)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actionview (7.0.4.3)
-      activesupport (= 7.0.4.3)
+    actionview (7.0.5)
+      activesupport (= 7.0.5)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activesupport (7.0.4.3)
+    activesupport (7.0.5)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -37,7 +37,7 @@ GEM
       websocket-driver (>= 0.6.5)
     ast (2.4.2)
     builder (3.2.4)
-    capybara (3.39.1)
+    capybara (3.39.2)
       addressable
       matrix
       mini_mime (>= 0.1.3)
@@ -48,59 +48,62 @@ GEM
       xpath (~> 3.2)
     concurrent-ruby (1.2.2)
     crass (1.0.6)
+    curb (1.0.5)
     dotenv (2.7.6)
     erubi (1.12.0)
     ethon (0.16.0)
       ffi (>= 1.15.0)
     ffi (1.15.5)
-    i18n (1.13.0)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
     json (2.6.3)
-    loofah (2.21.2)
+    loofah (2.21.3)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     matrix (0.4.2)
     method_source (1.0.0)
     mini_mime (1.1.2)
     minitest (5.18.0)
-    nokogiri (1.14.4-arm64-darwin)
+    nokogiri (1.15.2-arm64-darwin)
       racc (~> 1.4)
-    oj (3.14.3)
+    oj (3.15.0)
     parallel (1.23.0)
-    parser (3.2.2.1)
+    parser (3.2.2.3)
       ast (~> 2.4.1)
+      racc
     public_suffix (5.0.1)
-    racc (1.6.2)
+    racc (1.7.0)
     rack (2.2.7)
     rack-test (2.1.0)
       rack (>= 1.3)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.5.0)
-      loofah (~> 2.19, >= 2.19.1)
-    railties (7.0.4.3)
-      actionpack (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
+    rails-html-sanitizer (1.6.0)
+      loofah (~> 2.21)
+      nokogiri (~> 1.14)
+    railties (7.0.5)
+      actionpack (= 7.0.5)
+      activesupport (= 7.0.5)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
       zeitwerk (~> 2.5)
     rainbow (3.1.1)
     rake (13.0.6)
-    regexp_parser (2.8.0)
+    regexp_parser (2.8.1)
     rexml (3.2.5)
-    rubocop (1.51.0)
+    rubocop (1.52.1)
       json (~> 2.3)
       parallel (~> 1.10)
-      parser (>= 3.2.0.0)
+      parser (>= 3.2.2.3)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
       rubocop-ast (>= 1.28.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.28.1)
+    rubocop-ast (1.29.0)
       parser (>= 3.2.1.0)
     rubocop-md (1.2.0)
       rubocop (>= 1.0)
@@ -108,7 +111,7 @@ GEM
       rubocop (>= 1.39, < 2.0)
     rubocop-packaging (0.5.2)
       rubocop (>= 1.33, < 2.0)
-    rubocop-performance (1.17.1)
+    rubocop-performance (1.18.0)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
     rubocop-rails (2.19.1)
@@ -126,9 +129,9 @@ GEM
       rubocop-rails (~> 2.0)
     ruby-progressbar (1.13.0)
     rubyzip (2.3.2)
-    selenium-devtools (0.113.0)
+    selenium-devtools (0.114.0)
       selenium-webdriver (~> 4.2)
-    selenium-webdriver (4.9.1)
+    selenium-webdriver (4.10.0)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
@@ -150,6 +153,7 @@ PLATFORMS
   arm64-darwin-22
 
 DEPENDENCIES
+  curb (~> 1.0, >= 1.0.5)
   dotenv (~> 2.7.6)
   minitest (~> 5.0)
   rake (~> 13.0)

data/lib/zorki/scrapers/post_scraper.rb

@@ -45,15 +45,17 @@ module Zorki
         end.first["userInteractionCount"]
 
         unless graphql_object["video"].empty?
-          video = graphql_object["video"].first["contentUrl"]
-          video_preview_image = graphql_object["video"].first["thumbnailUrl"]
+          video_url = graphql_object["video"].first["contentUrl"]
+          video = Zorki.retrieve_media(video_url)
+
+          video_preview_image_url = graphql_object["video"].first["thumbnailUrl"]
+          video_preview_image = Zorki.retrieve_media(video_preview_image_url)
         end
       else
         # We need to see if this is a single image post or a slideshow. We do that
         # by looking for a single image, if it's not there, we assume the alternative.
         graphql_object = graphql_object["data"]["xdt_api__v1__media__shortcode__web_info"]
 
-
         unless graphql_object["items"][0].has_key?("video_versions") && !graphql_object["items"][0]["video_versions"].nil?
           # Check if there is a slideshow or not
           unless graphql_object["items"][0].has_key?("carousel_media") && !graphql_object["items"][0]["carousel_media"].nil?

data/lib/zorki/scrapers/scraper.rb

@@ -7,6 +7,7 @@ require "selenium-webdriver"
 require "logger"
 require "debug"
 require "securerandom"
+require "selenium/webdriver/remote/http/curb"
 
 # 2022-06-07 14:15:23 WARN Selenium [DEPRECATION] [:browser_options] :options as a parameter for driver initialization is deprecated. Use :capabilities with an Array of value capabilities/options if necessary instead.
 
@@ -16,13 +17,14 @@ options.add_argument("--no-sandbox")
 options.add_argument("--disable-dev-shm-usage")
 options.add_argument("–-disable-blink-features=AutomationControlled")
 options.add_argument("--disable-extensions")
+options.add_argument("--enable-features=NetworkService,NetworkServiceInProcess")
 options.add_argument("user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 13_3_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36")
 options.add_preference "password_manager_enabled", false
 options.add_argument("--user-data-dir=/tmp/tarun_zorki_#{SecureRandom.uuid}")
 
 Capybara.register_driver :selenium_zorki do |app|
-  client = Selenium::WebDriver::Remote::Http::Default.new
-  client.read_timeout = 60  # Don't wait 60 seconds to return Net::ReadTimeoutError. We'll retry through Hypatia after 10 seconds
+  client = Selenium::WebDriver::Remote::Http::Curb.new
+  # client.read_timeout = 60  # Don't wait 60 seconds to return Net::ReadTimeoutError. We'll retry through Hypatia after 10 seconds
   Capybara::Selenium::Driver.new(app, browser: :chrome, options: options, http_client: client)
 end
 
@@ -94,7 +96,7 @@ module Zorki
       end
 
       # Now that the intercept is set up, we visit the page we want
-      visit(url)
+      page.driver.browser.navigate.to(url)
       # We wait until the correct intercept is processed or we've waited 60 seconds
       start_time = Time.now
       # puts "Waiting.... #{url}"
@@ -134,14 +136,16 @@ module Zorki
       options.add_argument("--disable-dev-shm-usage")
       options.add_argument("–-disable-blink-features=AutomationControlled")
       options.add_argument("--disable-extensions")
+      options.add_argument("--enable-features=NetworkService,NetworkServiceInProcess")
+
       options.add_argument("user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 13_3_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36")
       options.add_preference "password_manager_enabled", false
       options.add_argument("--user-data-dir=/tmp/tarun_zorki_#{SecureRandom.uuid}")
       # options.add_argument("--user-data-dir=/tmp/tarun")
 
       Capybara.register_driver :selenium do |app|
-        client = Selenium::WebDriver::Remote::Http::Default.new
-        client.read_timeout = 60  # Don't wait 60 seconds to return Net::ReadTimeoutError. We'll retry through Hypatia after 10 seconds
+        client = Selenium::WebDriver::Remote::Http::Curb.new
+        # client.read_timeout = 60  # Don't wait 60 seconds to return Net::ReadTimeoutError. We'll retry through Hypatia after 10 seconds
         Capybara::Selenium::Driver.new(app, browser: :chrome, options: options, http_client: client)
       end
 
@@ -153,7 +157,12 @@ module Zorki
       page.quit
 
       # Check if we're on a Instagram page already, if not visit it.
-      visit ("https://instagram.com") unless page.driver.browser.current_url.include? "instagram.com"
+      unless page.driver.browser.current_url.include? "instagram.com"
+        # There seems to be a bug in the Linux ARM64 version of chromedriver where this will properly
+        # navigate but then timeout, crashing it all up. So instead we check and raise the error when
+        # that then fails again.
+        page.driver.browser.navigate.to("https://instagram.com")
+      end
 
       # We don't have to login if we already are
       begin

data/lib/zorki/scrapers/user_scraper.rb

@@ -19,6 +19,7 @@ module Zorki
       login
 
       graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "?username=")
+      graphql_script = graphql_script.first if graphql_script.class == Array
 
       if graphql_script.has_key?("author") && !graphql_script["author"].nil?
         user = graphql_script["author"]
@@ -28,26 +29,28 @@ module Zorki
         raise Zorki::Error unless username == scraped_username
 
         number_of_posts = graphql_script["interactionStatistic"].select do |stat|
-          stat["interactionType"] == "https://schema.org/FilmAction"
+          ["https://schema.org/FilmAction", "http://schema.org/WriteAction"].include?(stat["interactionType"])
         end.first
 
         number_of_followers = graphql_script["interactionStatistic"].select do |stat|
           stat["interactionType"] == "http://schema.org/FollowAction"
         end.first
 
-        profile_image_url = user["image"]
-        {
-          name: user["name"],
-          username: username,
-          number_of_posts: Integer(number_of_posts["userInteractionCount"]),
-          number_of_followers: Integer(number_of_followers["userInteractionCount"]),
-          # number_of_following: user["edge_follow"]["count"],
-          verified: user["is_verified"], # todo
-          profile: graphql_script["description"],
-          profile_link: user["sameAs"],
-          profile_image: Zorki.retrieve_media(profile_image_url),
-          profile_image_url: profile_image_url
-        }
+        begin
+          profile_image_url = user["image"]
+          {
+            name: user["name"],
+            username: username,
+            number_of_posts: Integer(number_of_posts["userInteractionCount"]),
+            number_of_followers: Integer(number_of_followers["userInteractionCount"]),
+            # number_of_following: user["edge_follow"]["count"],
+            verified: user["is_verified"], # todo
+            profile: graphql_script["description"],
+            profile_link: user["sameAs"],
+            profile_image: Zorki.retrieve_media(profile_image_url),
+            profile_image_url: profile_image_url
+          }
+        end
       else
         user = graphql_script["data"]["user"]
 

data/lib/zorki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zorki
-  VERSION = "0.1.1"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zorki
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - Christopher Guess
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-23 00:00:00.000000000 Z
+date: 2023-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara