morris 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 07cf83296ce2b6ecdb52c4afe1b92d63fe861e57327462886cc1ba5f144e2ce5
-  data.tar.gz: 72d9d028fa9ee0490be70c41784de98f495774eaf25d17cb42865f182ca6c1a8
+  metadata.gz: 6532b430aabd03227472ecbffca87240f944fe0fb5308f5436b033a54fb9879f
+  data.tar.gz: af5ec6041a761da92ed19d9248aff5a3818c62f0f156d0861fd41c8c972f9a36
 SHA512:
-  metadata.gz: d4af8e95bcd1b2ff5c9008f8f8a6ade8bc6e366b6ec96c349abb4298d9a4bf752fcda8bb7dbc7729c3ecd23b91d33dd85b5f3fa0696f21ec5016d00c53f72f17
-  data.tar.gz: f4eb227b310d4ef27bfd346c09f4604bfa29e038346429e1ab2aa5cdfe56f20226347e12e6b3f8feab4e34ce5792ae9e43f46be3e3e789df5956da82230bdd47
+  metadata.gz: cc49d05f28a5623c67ecb8e89c192991365d065f22963be72c731723697652ce8de7891278bdcbb4b70294268c131d27a4ab655b7193b967d5733c2bd8c2b5d2
+  data.tar.gz: 1ccf7a3aa96ad633958e793474dccba13fa743a52fcefdbf2929f11fe3fa2e2810dffd79b7e150bc2aebf736203dafedaabe3ea91f8526e72eaab5e6f2701c6c

data/lib/morris/scrapers/post_scraper.rb

@@ -15,11 +15,30 @@ module Morris
 
       Capybara.app_host = "https://tiktok.com"
 
+      # Clean up the url
+      uri = URI.parse(url)
+
+      url = "#{uri.scheme}://#{uri.host}#{uri.path}"
+
       # Get the page
-      visit(url)
+      begin
+        visit(url)
+      rescue Addressable::URI::InvalidURIError
+        raise Morris::ContentUnavailableError.new
+      end
 
       # Grab the JSON
       element = page.all(:xpath, '//*[@id="__UNIVERSAL_DATA_FOR_REHYDRATION__"]', visible: false).first
+
+      # If the element is not found, raise a ContentUnavailableError
+      if element.nil?
+        begin
+          page.find(class: "not-found")
+        rescue Capybara::ElementNotFound
+          raise Morris::ContentUnavailableError.new
+        end
+      end
+
       text = element.text(:all) # Gotta get the hiddent text of the element
       json = JSON.parse(text)
 

data/lib/morris/scrapers/scraper.rb

@@ -21,6 +21,7 @@ options.add_argument("--enable-features=NetworkService,NetworkServiceInProcess")
 options.add_argument("user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 13_3_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36")
 options.add_preference "password_manager_enabled", false
 options.add_argument("--user-data-dir=/tmp/tarun_morris_#{SecureRandom.uuid}")
+options.add_argument("--mute-audio")
 
 Capybara.register_driver :selenium_morris do |app|
   client = Selenium::WebDriver::Remote::Http::Curb.new

data/lib/morris/scrapers/video_scraper.rb

@@ -23,7 +23,8 @@ module Morris
 
       start_time = Time.now
       filename = "#{Morris.temp_storage_location}/morris_media_#{SecureRandom.uuid}.mp4"
-      line = Terrapin::CommandLine.new("yt-dlp", "-f :filetype -o :filename :url")
+      # line = Terrapin::CommandLine.new("yt-dlp", "-f :filetype -o :filename :url")
+      line = Terrapin::CommandLine.new("yt-dlp", "-f 0 -o :filename :url")
 
       line.run(filename: filename,
                filetype: "mp4",

data/lib/morris/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Morris
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: morris
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Christopher Guess
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-03-05 00:00:00.000000000 Z
+date: 2024-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -129,7 +129,6 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".byebug_history"
 - ".rubocop.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md

data/.byebug_history

@@ -1,85 +0,0 @@
-exit
-content_element["createTime"]
-c
-username
-c
-up
-n
-username
-quit
-Time.at(content_element["createTime"].to_i)
-content_element["createTime"]
-exit
-pp json["__DEFAULT_SCOPE__"].keys
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]["author"]
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"].keys
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]["video"]["cover"]
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]["video"].keys
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]["video"]
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]["digged"]
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]["textExtra"]
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]["originalItem"]
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]["stats"]["diggCount"]
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]["stats"]
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]["challenges"
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]["id"]
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]["desc"]
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]["contents"]
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"].keys
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"].keys
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]
-pp json["__DEFAULT_SCOPE__"]["webapp.video-detail"].keys
-pp json["__DEFAULT_SCOPE__"].keys
-pp json
-text
-json
-json["__DEFAULT_SCOPE__"]
-json["__DEFAULT_SCOPE"]
-json.keys
-json
-c
-quit
-page.all(:xpath, '//*[@id="__UNIVERSAL_DATA_FOR_REHYDRATION__"]', visible: false).first.text(:all)
-page.all(:xpath, '//*[@id="__UNIVERSAL_DATA_FOR_REHYDRATION__"]', visible: false).first.innerText
-page.all(:xpath, '//*[@id="__UNIVERSAL_DATA_FOR_REHYDRATION__"]', visible: false).first.text
-page.all(:xpath, '//*[@id="__UNIVERSAL_DATA_FOR_REHYDRATION__"]', visible: false).first
-page.all(:xpath, '//*[@id="__UNIVERSAL_DATA_FOR_REHYDRATION__"]', visible: false).text
-page.all(:xpath, '//*[@id="__UNIVERSAL_DATA_FOR_REHYDRATION__"]', visible: false).innerText
-page.all(:xpath, '//*[@id="__UNIVERSAL_DATA_FOR_REHYDRATION__"]', visible: false)
-page.all(:xpath, "*[@id='__UNIVERSAL_DATA_FOR_REHYDRATION__']", visible: false)
-c
-page.all(:xpath, "*[@id='__UNIVERSAL_DATA_FOR_REHYDRATION__']", visible: false)
-page.all(:xpath, "script[@id='__UNIVERSAL_DATA_FOR_REHYDRATION__']", visible: false)
-page.find(:xpath, "script[@id='__UNIVERSAL_DATA_FOR_REHYDRATION__']", visible: false)
-page.find(:xpath, "script[@id='__UNIVERSAL_DATA_FOR_REHYDRATION__']")
-exit
-File.size(result.video_file) > 500000
-File.size(result.video_file)
-File.exist?(result.video_file)
-result.video_file
-result[:path]
-result
-exit
-RUBY_ENV
-video_hash
-up
-exit
-video_hash.transform_keys(&:to_s)
-video_hash.strinify_keys
-video_hash
-up
-exit
-video_hash
-@id
-@iid
-up
-@id
-line
-c
-@id
-e
-c
-e
-exit
-e
-c