mosquito-scrape 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b1a8c44c0a42c50a30f8e2ef7afed0ef753b08c5391a3b2d32973bdedee1f81
-  data.tar.gz: d7e0e1a229e9e7aae15e0644ca7f0f0655490c61e5df7d3ce5247b6442d75d4d
+  metadata.gz: 7c00016d7bb5c8cba182ee58439c0e4291abdf057bdfd82210426549c57a97ff
+  data.tar.gz: a1c4a7bc39cfc67760ef425f45d8ac5ed04f123086692b0ce0bd762f104baa18
 SHA512:
-  metadata.gz: d853aa27352a4c5951b3be15770fe655577299953894a04b9106b64c8bd7a4ff60f74d5e84d159769a278122f7a5cbb129d8d4ec0aec71d3bbb205ba2bd5069f
-  data.tar.gz: acc3c469886dcd8658d841595f2ce7dc1469fb23696e3a82f28eecdb8b261e73e8960029df60ae41d57616fb3c7ad5468aa7cf6255a173989eafb720d32a2af6
+  metadata.gz: a7818a6933e022965d43028fb4706be4694134088004aea6c57dc11040346ddfa022378b8e2a9da9c0e343985d63d5aa7222d0ce66747d10edd09defc180aac9
+  data.tar.gz: 0564c11d892c5111582c165f858441678168219c32ecc20d63b8eb67cb0e9d655beab95881186f456ea2ccf0ad46fab8d6a6a40cfa376c8a1b82db2733b6ad55

data/lib/mosquito/scrapers/tweet_scraper.rb

@@ -42,26 +42,29 @@ module Mosquito
       video_preview_image = nil
       video_file_type = nil
 
-      # Single image
-      images.concat(doc.xpath("//a[contains(@class, 'still-image')][1]/href"))
+      # # Single image
+      # image_url = doc.xpath("//div[contains(@class, 'main-tweet')]/div/div/div/div/div/a[contains(@class, 'still-image')]/@href").first&.content
+      # images << Mosquito.retrieve_media("#{Capybara.app_host}#{image_url}") unless image_url.nil?
+
+      # debugger
 
       # Slideshow
       nodes = doc.xpath("//div[contains(@class, 'main-tweet')]/div/div/div[contains(@class, 'attachments')]/div[contains(@class, 'gallery-row')]/div/a/@href")
-      images.concat(nodes.map { |node| "#{Capybara.app_host}#{node.value}" })
+      images.concat(nodes.map { |node| Mosquito.retrieve_media("#{Capybara.app_host}#{node.value}") })
 
       # Video
       nodes = doc.xpath("//div[contains(@class, 'main-tweet')]/div/div/div[contains(@class, 'attachments')]/div[contains(@class, 'gallery-video')]/div/video")
       unless nodes.empty?
-        video_preview_image = nodes.first["poster"]
-        videos.concat(nodes.map { |node| "#{Capybara.app_host}#{node.xpath("//source").first["src"]}" })
+        video_preview_image = Mosquito.retrieve_media("#{Capybara.app_host}#{nodes.first["poster"]}", extension: ".jpg")
+        videos.concat(nodes.map { |node| Mosquito.retrieve_media(node.xpath("//source").first["src"]) })
         video_file_type = "video" # This is always video now, sing a gif isn't displayed differently
       end
 
       # GIF
       nodes = doc.xpath("//div[contains(@class, 'main-tweet')]/div/div/div[contains(@class, 'attachments')]/div[contains(@class, 'gallery-gif')]/div/video")
       unless nodes.empty?
-        video_preview_image = nodes.first["poster"]
-        videos.concat(nodes.map { |node| "#{Capybara.app_host}#{node.xpath("//source[1]/source/@src").first&.content}" })
+        video_preview_image = Mosquito.retrieve_media(nodes.first["poster"], extension: ".jpg")
+        videos.concat(nodes.map { |node| Mosquito.retrieve_media("#{Capybara.app_host}#{node.xpath("//source[1]/source/@src").first&.content}") })
         video_file_type = "gif"
       end
 

data/lib/mosquito/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mosquito
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/mosquito.rb

@@ -49,20 +49,23 @@ module Mosquito
 
   # Get media from a URL and save to a temp folder set in the configuration under
   # temp_storage_location
-  def self.retrieve_media(url)
+  def self.retrieve_media(url, extension: nil)
+    return "" if url.nil?
     return "" if !Mosquito.save_media
 
     response = Typhoeus.get(url)
 
     # Get the file extension if it's in the file
-    extension = url.split(".").last
-
-    # Do some basic checks so we just empty out if there's something weird in the file extension
-    # that could do some harm.
-    if extension.length.positive?
-      extension = extension[0...extension.index("?")]
-      extension = nil unless /^[a-zA-Z0-9]+$/.match?(extension)
-      extension = ".#{extension}" unless extension.nil?
+    if extension.nil?
+      extension = url.split(".").last
+
+      # Do some basic checks so we just empty out if there's something weird in the file extension
+      # that could do some harm.
+      if extension.length.positive?
+        extension = extension[0...extension.index("?")]
+        extension = nil unless /^[a-zA-Z0-9]+$/.match?(extension)
+        extension = ".#{extension}" unless extension.nil?
+      end
     end
 
     temp_file_name = "#{Mosquito.temp_storage_location}/#{SecureRandom.uuid}#{extension}"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mosquito-scrape
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Christopher Guess