zorki 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e9aca1027df18f607fac0616a75a4b8ac11728cbac5de9205f014c57306f82e
-  data.tar.gz: c22375fd87090060642780ae2ec472d505f66c7ed1579d35d8eb0e9fe02bd976
+  metadata.gz: 6dd3c28132011c0d9d42875803face311af01a50bbbfa9cf4f07ca89a63029d4
+  data.tar.gz: 91fa19abbd41551e4cb55ff34ce3977795903312c06506bf9bb450796cca7189
 SHA512:
-  metadata.gz: b3e3117ca0903a23fd2303ec938d70d1d8dd8f82eeb510019832e25e286a4c5ae665c707cc62523dedf9617de9239072aa02d8877fb8e2fae2ea4f85ad14bbc9
-  data.tar.gz: 7a2b9d079041484f5553bf5be6fa391bbeb20c8660fb07df69cd7b6feb28d8f7d315e22353d89ff1ccedf7304e39dfbab9f8f18e9f25ad44cd7067e473d515d1
+  metadata.gz: '087a14d77466f9b5f70014f5f09ae97542d2b8083499917d0397815dd4c298da07da9c4b28fb31e6f7c1949a356e9c4d32b8bb8e7551054ca1fe5c37c972515b'
+  data.tar.gz: 5b46e4e7edb229d89ff5f9bb740dac2959c314e09cf5b9a770c8b16779509266a8bd06d38b1a1208f547cfef22e10fd139b0f32dbea9c89dba5413874399743a

data/lib/zorki/scrapers/post_scraper.rb

@@ -23,6 +23,8 @@ module Zorki
         "data,xdt_api__v1__media__shortcode__web_info,items"
       )
 
+      graphql_object = graphql_object.first if graphql_object.kind_of?(Array)
+
       # For pages that have been marked misinfo the structure is very different than not
       # If it is a clean post then it's just a schema.org thing, but if it's misinfo it's the old
       # way of deeply nested stuff.
@@ -54,7 +56,8 @@ module Zorki
       else
         # We need to see if this is a single image post or a slideshow. We do that
         # by looking for a single image, if it's not there, we assume the alternative.
-        graphql_object = graphql_object["data"]["xdt_api__v1__media__shortcode__web_info"]
+        # debugger
+        # graphql_object = graphql_object["data"]["xdt_api__v1__media__shortcode__web_info"]
 
         unless graphql_object["items"][0].has_key?("video_versions") && !graphql_object["items"][0]["video_versions"].nil?
           # Check if there is a slideshow or not

data/lib/zorki/scrapers/scraper.rb

@@ -5,9 +5,9 @@ require "dotenv/load"
 require "oj"
 require "selenium-webdriver"
 require "logger"
-require "debug"
 require "securerandom"
 require "selenium/webdriver/remote/http/curb"
+require "debug"
 
 # 2022-06-07 14:15:23 WARN Selenium [DEPRECATION] [:browser_options] :options as a parameter for driver initialization is deprecated. Use :capabilities with an Array of value capabilities/options if necessary instead.
 
@@ -112,12 +112,27 @@ module Zorki
       # TODO: put this before the whole load loop
       if response_body.nil?
         doc = Nokogiri::HTML(page.driver.browser.page_source)
-        elements = doc.search("script").find_all do |e|
-          e.attributes.has_key?("type") && e.attributes["type"].value == "application/ld+json"
-        end
+        # elements = doc.search("script").find_all do |e|
+        #   e.attributes.has_key?("type") && e.attributes["type"].value == "application/ld+json"
+        # end
+
+        elements = doc.search("script").map do |element|
+          element_json = nil
+          begin
+            element_json = JSON.parse(element)
+
+            element_json = element_json["require"].first.last.first["__bbox"]["require"].first.last.last["__bbox"]["result"]["data"]["xdt_api__v1__media__shortcode__web_info"]
+          rescue StandardError => e
+            next
+          end
+
+          element_json
+        end.compact
 
-        raise ContentUnavailableError if elements&.empty?
-        return Oj.load(elements.first.text)
+        if elements&.empty?
+          raise ContentUnavailableError
+        end
+        return elements
       end
 
       raise ContentUnavailableError if response_body.nil?

data/lib/zorki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zorki
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zorki
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Christopher Guess
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-12 00:00:00.000000000 Z
+date: 2023-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara