zorki 0.1.2 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e866c6873fc604f1720faeaf9425ff0557a6ef9947e9a9e4f657cc255a8d4665
-  data.tar.gz: e97fa8e84dd02edc9dd47beb9d66fe2b3cab4704297f16005dc09adf68eeb2e4
+  metadata.gz: 6dd3c28132011c0d9d42875803face311af01a50bbbfa9cf4f07ca89a63029d4
+  data.tar.gz: 91fa19abbd41551e4cb55ff34ce3977795903312c06506bf9bb450796cca7189
 SHA512:
-  metadata.gz: f5d6f36fede35c5d54289f6f7a41098e5374f74e8bbbf18ebd66486e4c049fdd064e063a7787000ed5997a9460496d9c5310795585a9098b1bfcbf891c7277e6
-  data.tar.gz: 8a79812eb8d312fef165868bbaad85cc4a7cd0f67ed04fb3e7cb0246e417ec5eefb89a54d3eec7ff78b3a3b38a9df89ae8c82423ea7fbb4ab04d160c86b397f5
+  metadata.gz: '087a14d77466f9b5f70014f5f09ae97542d2b8083499917d0397815dd4c298da07da9c4b28fb31e6f7c1949a356e9c4d32b8bb8e7551054ca1fe5c37c972515b'
+  data.tar.gz: 5b46e4e7edb229d89ff5f9bb740dac2959c314e09cf5b9a770c8b16779509266a8bd06d38b1a1208f547cfef22e10fd139b0f32dbea9c89dba5413874399743a

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    zorki (0.1.1)
+    zorki (0.1.2)
       apparition
       capybara
       oj
@@ -12,20 +12,20 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (7.0.4.3)
-      actionview (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
-      rack (~> 2.0, >= 2.2.0)
+    actionpack (7.0.5)
+      actionview (= 7.0.5)
+      activesupport (= 7.0.5)
+      rack (~> 2.0, >= 2.2.4)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actionview (7.0.4.3)
-      activesupport (= 7.0.4.3)
+    actionview (7.0.5)
+      activesupport (= 7.0.5)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activesupport (7.0.4.3)
+    activesupport (7.0.5)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -37,7 +37,7 @@ GEM
       websocket-driver (>= 0.6.5)
     ast (2.4.2)
     builder (3.2.4)
-    capybara (3.39.1)
+    capybara (3.39.2)
       addressable
       matrix
       mini_mime (>= 0.1.3)
@@ -54,54 +54,56 @@ GEM
     ethon (0.16.0)
       ffi (>= 1.15.0)
     ffi (1.15.5)
-    i18n (1.13.0)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
     json (2.6.3)
-    loofah (2.21.2)
+    loofah (2.21.3)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     matrix (0.4.2)
     method_source (1.0.0)
     mini_mime (1.1.2)
     minitest (5.18.0)
-    nokogiri (1.14.4-arm64-darwin)
+    nokogiri (1.15.2-arm64-darwin)
       racc (~> 1.4)
-    oj (3.14.3)
+    oj (3.15.0)
     parallel (1.23.0)
-    parser (3.2.2.1)
+    parser (3.2.2.3)
       ast (~> 2.4.1)
+      racc
     public_suffix (5.0.1)
-    racc (1.6.2)
+    racc (1.7.0)
     rack (2.2.7)
     rack-test (2.1.0)
       rack (>= 1.3)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.5.0)
-      loofah (~> 2.19, >= 2.19.1)
-    railties (7.0.4.3)
-      actionpack (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
+    rails-html-sanitizer (1.6.0)
+      loofah (~> 2.21)
+      nokogiri (~> 1.14)
+    railties (7.0.5)
+      actionpack (= 7.0.5)
+      activesupport (= 7.0.5)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
       zeitwerk (~> 2.5)
     rainbow (3.1.1)
     rake (13.0.6)
-    regexp_parser (2.8.0)
+    regexp_parser (2.8.1)
     rexml (3.2.5)
-    rubocop (1.51.0)
+    rubocop (1.52.1)
       json (~> 2.3)
       parallel (~> 1.10)
-      parser (>= 3.2.0.0)
+      parser (>= 3.2.2.3)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
       rubocop-ast (>= 1.28.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.28.1)
+    rubocop-ast (1.29.0)
       parser (>= 3.2.1.0)
     rubocop-md (1.2.0)
       rubocop (>= 1.0)
@@ -109,7 +111,7 @@ GEM
       rubocop (>= 1.39, < 2.0)
     rubocop-packaging (0.5.2)
       rubocop (>= 1.33, < 2.0)
-    rubocop-performance (1.17.1)
+    rubocop-performance (1.18.0)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
     rubocop-rails (2.19.1)
@@ -127,9 +129,9 @@ GEM
       rubocop-rails (~> 2.0)
     ruby-progressbar (1.13.0)
     rubyzip (2.3.2)
-    selenium-devtools (0.113.0)
+    selenium-devtools (0.114.0)
       selenium-webdriver (~> 4.2)
-    selenium-webdriver (4.9.1)
+    selenium-webdriver (4.10.0)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)

data/lib/zorki/scrapers/post_scraper.rb

@@ -23,6 +23,8 @@ module Zorki
         "data,xdt_api__v1__media__shortcode__web_info,items"
       )
 
+      graphql_object = graphql_object.first if graphql_object.kind_of?(Array)
+
       # For pages that have been marked misinfo the structure is very different than not
       # If it is a clean post then it's just a schema.org thing, but if it's misinfo it's the old
       # way of deeply nested stuff.
@@ -54,7 +56,8 @@ module Zorki
       else
         # We need to see if this is a single image post or a slideshow. We do that
         # by looking for a single image, if it's not there, we assume the alternative.
-        graphql_object = graphql_object["data"]["xdt_api__v1__media__shortcode__web_info"]
+        # debugger
+        # graphql_object = graphql_object["data"]["xdt_api__v1__media__shortcode__web_info"]
 
         unless graphql_object["items"][0].has_key?("video_versions") && !graphql_object["items"][0]["video_versions"].nil?
           # Check if there is a slideshow or not

data/lib/zorki/scrapers/scraper.rb

@@ -5,9 +5,9 @@ require "dotenv/load"
 require "oj"
 require "selenium-webdriver"
 require "logger"
-require "debug"
 require "securerandom"
 require "selenium/webdriver/remote/http/curb"
+require "debug"
 
 # 2022-06-07 14:15:23 WARN Selenium [DEPRECATION] [:browser_options] :options as a parameter for driver initialization is deprecated. Use :capabilities with an Array of value capabilities/options if necessary instead.
 
@@ -112,12 +112,27 @@ module Zorki
       # TODO: put this before the whole load loop
       if response_body.nil?
         doc = Nokogiri::HTML(page.driver.browser.page_source)
-        elements = doc.search("script").find_all do |e|
-          e.attributes.has_key?("type") && e.attributes["type"].value == "application/ld+json"
-        end
+        # elements = doc.search("script").find_all do |e|
+        #   e.attributes.has_key?("type") && e.attributes["type"].value == "application/ld+json"
+        # end
+
+        elements = doc.search("script").map do |element|
+          element_json = nil
+          begin
+            element_json = JSON.parse(element)
+
+            element_json = element_json["require"].first.last.first["__bbox"]["require"].first.last.last["__bbox"]["result"]["data"]["xdt_api__v1__media__shortcode__web_info"]
+          rescue StandardError => e
+            next
+          end
+
+          element_json
+        end.compact
 
-        raise ContentUnavailableError if elements&.empty?
-        return Oj.load(elements.first.text)
+        if elements&.empty?
+          raise ContentUnavailableError
+        end
+        return elements
       end
 
       raise ContentUnavailableError if response_body.nil?

data/lib/zorki/scrapers/user_scraper.rb

@@ -19,6 +19,7 @@ module Zorki
       login
 
       graphql_script = get_content_of_subpage_from_url("https://instagram.com/#{username}/", "?username=")
+      graphql_script = graphql_script.first if graphql_script.class == Array
 
       if graphql_script.has_key?("author") && !graphql_script["author"].nil?
         user = graphql_script["author"]
@@ -28,26 +29,28 @@ module Zorki
         raise Zorki::Error unless username == scraped_username
 
         number_of_posts = graphql_script["interactionStatistic"].select do |stat|
-          stat["interactionType"] == "https://schema.org/FilmAction"
+          ["https://schema.org/FilmAction", "http://schema.org/WriteAction"].include?(stat["interactionType"])
         end.first
 
         number_of_followers = graphql_script["interactionStatistic"].select do |stat|
           stat["interactionType"] == "http://schema.org/FollowAction"
         end.first
 
-        profile_image_url = user["image"]
-        {
-          name: user["name"],
-          username: username,
-          number_of_posts: Integer(number_of_posts["userInteractionCount"]),
-          number_of_followers: Integer(number_of_followers["userInteractionCount"]),
-          # number_of_following: user["edge_follow"]["count"],
-          verified: user["is_verified"], # todo
-          profile: graphql_script["description"],
-          profile_link: user["sameAs"],
-          profile_image: Zorki.retrieve_media(profile_image_url),
-          profile_image_url: profile_image_url
-        }
+        begin
+          profile_image_url = user["image"]
+          {
+            name: user["name"],
+            username: username,
+            number_of_posts: Integer(number_of_posts["userInteractionCount"]),
+            number_of_followers: Integer(number_of_followers["userInteractionCount"]),
+            # number_of_following: user["edge_follow"]["count"],
+            verified: user["is_verified"], # todo
+            profile: graphql_script["description"],
+            profile_link: user["sameAs"],
+            profile_image: Zorki.retrieve_media(profile_image_url),
+            profile_image_url: profile_image_url
+          }
+        end
       else
         user = graphql_script["data"]["user"]
 

data/lib/zorki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zorki
-  VERSION = "0.1.2"
+  VERSION = "0.1.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zorki
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.4
 platform: ruby
 authors:
 - Christopher Guess
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-27 00:00:00.000000000 Z
+date: 2023-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara