zorki 0.1.29 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 684dbf1fee604da4f2ab9687223bf3aea6f1ebad0a4f9e16ef7c80dafa645ac0
-  data.tar.gz: ab628433652062674b97318864663082cf5c7e623d32ff4ec87494d54df79788
+  metadata.gz: 360620c86ba91caf49a0e41a0362654620f60da706c761f3f9429757f570c7a4
+  data.tar.gz: 442628f21ce102c28e22f2aa0cc9243fe1c836267a6901cd53c5f094fe778444
 SHA512:
-  metadata.gz: cfec123b6b4f1abb61c77c963b4162a9ef87273e3c3bc1dc7a67be34a5af5c38dbc70f4a43d3ae5690a3d5655a54d3081545352eb0768b926d598a575c1c3654
-  data.tar.gz: d7cc5599d995dcacc5d38dc60669f67f75d6005f282d9013ceb12bdb463b87ddf78eb9a156bf5e52ad0d3f464f1adea89abc2cc0ae371b4cbcbb981b72124593
+  metadata.gz: bb9d411327c46d63229dfc8e43cfcfb43d14a4b6a87615d07aa7871a29a9d0fe7390ef018134eb358314300fa1a67c0e976a43fd0357e7eea0efac4b4afede30
+  data.tar.gz: 5ef69f54c80920ebfacf7520c851b06649b81262df809041bb6e97a981e8fd237ccea5b7d8972b4332e2a78fe25330171af9d0f62c9dfc2ccf2029cd0409044a

data/lib/zorki/scrapers/scraper.rb

@@ -81,7 +81,6 @@ module Zorki
           puts response.body
           puts "***********************************************************"
 
-          # responses << response
           # Check if not a CORS prefetch and finish up if not
           if !response.body&.empty? && response.body
             check_passed = true
@@ -103,6 +102,7 @@ module Zorki
               puts "checking FAILED request: #{request.url}"
               puts response.body
               puts "***********************************************************"
+              next
             end
 
             response_body = response.body if check_passed == true
@@ -110,10 +110,10 @@ module Zorki
         end
       rescue Selenium::WebDriver::Error::WebDriverError
         # Eat them
-      rescue StandardError => e
-        puts "***********************************************************"
-        puts "Error in intercept: #{e}"
-        puts "***********************************************************"
+        # rescue StandardError => e
+        # puts "***********************************************************"
+        # puts "Error in intercept: #{e}"
+        # puts "***********************************************************"
       end
 
       # Now that the intercept is set up, we visit the page we want
@@ -132,7 +132,6 @@ module Zorki
       # If this is a page that has not been marked as misinfo we can just pull the data
       # TODO: put this before the whole load loop
       if response_body.nil?
-
         doc = Nokogiri::HTML(page.driver.browser.page_source)
         # elements = doc.search("script").find_all do |e|
         #   e.attributes.has_key?("type") && e.attributes["type"].value == "application/ld+json"
@@ -141,7 +140,7 @@ module Zorki
         elements = doc.search("script").filter_map do |element|
           parsed_element_json = nil
           begin
-            element_json = OJ.load(element.text)
+            element_json = Oj.load(element.text)
 
             # if element.text.include?("jokoy.komi.io")
             # debugger
@@ -155,6 +154,10 @@ module Zorki
 
             parsed_element_json = element_json["require"].last.last.first["__bbox"]["require"].first.last.last["__bbox"]["result"]["data"]["xdt_api__v1__media__shortcode__web_info"]
           rescue StandardError
+            # puts "***********************************************************"
+            # puts "Error in parsing JSON: #{e}"
+            # puts e.backtrace
+            # puts "***********************************************************"
             next
           end
 

data/lib/zorki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zorki
-  VERSION = "0.1.29"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: zorki
 version: !ruby/object:Gem::Version
-  version: 0.1.29
+  version: 0.2.1
 platform: ruby
 authors:
 - Christopher Guess