zorki 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e9aca1027df18f607fac0616a75a4b8ac11728cbac5de9205f014c57306f82e
-  data.tar.gz: c22375fd87090060642780ae2ec472d505f66c7ed1579d35d8eb0e9fe02bd976
+  metadata.gz: 174766669fddfafe37892c58e5caa7706e9016244bda0621353178700631c8ef
+  data.tar.gz: 0b061de3dce20457b29ffbae043bb12a35a6deabca528effa1134f3757610403
 SHA512:
-  metadata.gz: b3e3117ca0903a23fd2303ec938d70d1d8dd8f82eeb510019832e25e286a4c5ae665c707cc62523dedf9617de9239072aa02d8877fb8e2fae2ea4f85ad14bbc9
-  data.tar.gz: 7a2b9d079041484f5553bf5be6fa391bbeb20c8660fb07df69cd7b6feb28d8f7d315e22353d89ff1ccedf7304e39dfbab9f8f18e9f25ad44cd7067e473d515d1
+  metadata.gz: cf2b8fc81c18bd4a4e4095c81f1ea6832a0c17a625eb802e13a0cc37ea027a912992158d7b9aba199e0225d46020dd098410a2ab4d61b590ab37894e118127a5
+  data.tar.gz: 6d380eb69c798c66e0a77b504ca2b7f6494afb3c1ea9ea77eacd6868f0476c60003c9f1f46dfc8683c2290833715ece072f40de48b395ab93369e769be945227

data/lib/zorki/monkeypatch.rb

@@ -40,7 +40,9 @@ module SeleniumMonkeypatch
       data = { method: method, params: params.compact }
       data[:sessionId] = @session_id if @session_id
       message = @ws.send_cmd(**data)
-      raise Error::WebDriverError, error_message(message["error"]) if message["error"] && (method != "Fetch.continueRequest")
+      if message.nil? == false && message["error"] && (method != "Fetch.continueRequest")
+        raise Error::WebDriverError, error_message(message["error"])
+      end
 
       message
     end

data/lib/zorki/scrapers/post_scraper.rb

@@ -23,6 +23,8 @@ module Zorki
         "data,xdt_api__v1__media__shortcode__web_info,items"
       )
 
+      graphql_object = graphql_object.first if graphql_object.kind_of?(Array)
+
       # For pages that have been marked misinfo the structure is very different than not
       # If it is a clean post then it's just a schema.org thing, but if it's misinfo it's the old
       # way of deeply nested stuff.
@@ -54,7 +56,8 @@ module Zorki
       else
         # We need to see if this is a single image post or a slideshow. We do that
         # by looking for a single image, if it's not there, we assume the alternative.
-        graphql_object = graphql_object["data"]["xdt_api__v1__media__shortcode__web_info"]
+        # debugger
+        # graphql_object = graphql_object["data"]["xdt_api__v1__media__shortcode__web_info"]
 
         unless graphql_object["items"][0].has_key?("video_versions") && !graphql_object["items"][0]["video_versions"].nil?
           # Check if there is a slideshow or not

data/lib/zorki/scrapers/scraper.rb

@@ -5,9 +5,9 @@ require "dotenv/load"
 require "oj"
 require "selenium-webdriver"
 require "logger"
-require "debug"
 require "securerandom"
 require "selenium/webdriver/remote/http/curb"
+require "debug"
 
 # 2022-06-07 14:15:23 WARN Selenium [DEPRECATION] [:browser_options] :options as a parameter for driver initialization is deprecated. Use :capabilities with an Array of value capabilities/options if necessary instead.
 
@@ -112,12 +112,27 @@ module Zorki
       # TODO: put this before the whole load loop
       if response_body.nil?
         doc = Nokogiri::HTML(page.driver.browser.page_source)
-        elements = doc.search("script").find_all do |e|
-          e.attributes.has_key?("type") && e.attributes["type"].value == "application/ld+json"
-        end
+        # elements = doc.search("script").find_all do |e|
+        #   e.attributes.has_key?("type") && e.attributes["type"].value == "application/ld+json"
+        # end
+
+        elements = doc.search("script").map do |element|
+          element_json = nil
+          begin
+            element_json = JSON.parse(element)
+
+            element_json = element_json["require"].first.last.first["__bbox"]["require"].first.last.last["__bbox"]["result"]["data"]["xdt_api__v1__media__shortcode__web_info"]
+          rescue StandardError => e
+            next
+          end
+
+          element_json
+        end.compact
 
-        raise ContentUnavailableError if elements&.empty?
-        return Oj.load(elements.first.text)
+        if elements&.empty?
+          raise ContentUnavailableError
+        end
+        return elements
       end
 
       raise ContentUnavailableError if response_body.nil?

data/lib/zorki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zorki
-  VERSION = "0.1.3"
+  VERSION = "0.1.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zorki
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Christopher Guess
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-12 00:00:00.000000000 Z
+date: 2023-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara