pulse-downloader 0.1.12 → 0.1.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 17c6b467560188e6ccbf9549fe0be00188726daf87352d295788ccbbb986ead3
-  data.tar.gz: a88019ce7c0708767b8c76bb5796ec34950f1ddf02cd7f2e8b83cb09126bcbb8
+  metadata.gz: 0b50ec4293c1d064d672f7b9c8271837420deb866826a85d8c3016ca348bfd39
+  data.tar.gz: d4b35e8706bece03ba778320bb524fd8164f3b7e1933a3f79939b38abe5f9d20
 SHA512:
-  metadata.gz: b96d3ec2c08d342d4884c8d413d5133d1a71776f12e9dfc8c12829db9f1f0fc02f784f58f07481692f13b8d7a1a2dce66bd7e2c049077d1a337a676876a15c5b
-  data.tar.gz: b60ca5f1204ae2efd86c31c946dbcbff3d02c4279d83fd8f36c07d9909e4103db1df98b9009b5305b8271c3ba6c6fe7bda4daf708bb0505f15aab316e08af0a5
+  metadata.gz: e8626f65c533de9ca6422d6a1ea18c018e2ca23316ecd2678a5c9a733fe65d72eb1fd921e9a1215839e8501849cc36cb3c96d4e9cfa39e107fbc5422bc2d7be7
+  data.tar.gz: 70cc3bb87274c39ee93060292ee343144157806dd8900f78a564363bfad2a997182b2ae104ec0941603256fc8d38583d041f01e541d520ed169dc51989fb1f1b

data/lib/pulse/downloader/client.rb

@@ -8,6 +8,7 @@ module Pulse
 
       attr_reader :url,
         :file_type,
+        :scrape_images,
         :save_data,
         :save_path,
         :read_from_save_path,
@@ -17,7 +18,8 @@ module Pulse
         :report_time,
         :start_time,
         :end_time,
-        :progress_bar
+        :progress_bar,
+        :base_url
 
       # Does not continue downloads-
       # Will only save once the file has been downloaded in memory
@@ -29,6 +31,7 @@ module Pulse
       # TODO: lib/pulse/downloader/file_downloader.rb:13: warning: URI.escape is obsolete
       def initialize(url:,
         file_type:,
+        scrape_images: false,
         save_data: false,
         save_path: '',
         read_from_save_path: false,
@@ -40,6 +43,7 @@ module Pulse
 
         @url = url
         @file_type = file_type
+        @scrape_images = scrape_images
         @save_data = save_data
         @save_path = save_path
         @read_from_save_path = read_from_save_path
@@ -48,6 +52,8 @@ module Pulse
         @save_and_dont_return = save_and_dont_return
         @report_time = report_time
         @progress_bar = progress_bar
+
+        @base_url = get_base_url
       end
 
       def call!
@@ -73,6 +79,16 @@ module Pulse
 
       private
 
+      def get_base_url
+        url_breakdown = url.split('/')
+
+        if url_breakdown.first.include?('https')
+          url_breakdown[2]
+        else
+          url_breakdown.first
+        end
+      end
+
       def get_micro_second_time
         (Time.now.to_f * 1000).to_i
       end

data/lib/pulse/downloader/file_downloader.rb

@@ -68,33 +68,33 @@ module Pulse
       end
 
       def escape(str)
-        str.gsub("Space", "%20")
-        str.gsub("$", "\%24")
-        str.gsub("&", "\%26")
-        str.gsub("`", "\%60")
-        str.gsub(":", "\%3A")
-        str.gsub("<", "\%3C")
-        str.gsub(">", "\%3E")
-        str.gsub("[", "\%5B")
-        str.gsub("]", "\%5D")
-        str.gsub("{", "\%7B")
-        str.gsub("}", "\%7D")
-        str.gsub("“", "\%22")
-        str.gsub('"', "\%22")
-        str.gsub("+", "\%2B")
-        str.gsub("#", "\%23")
-        str.gsub("\%", "\%25")
-        str.gsub("@", "\%40")
-        str.gsub("/", "\%2F")
-        str.gsub(";", "\%3B")
-        str.gsub("=", "\%3D")
-        str.gsub("?", "\%3F")
-        str.gsub("\\", "\%5C")
-        str.gsub("^", "\%5E")
-        str.gsub("|", "\%7C")
-        str.gsub("~", "\%7E")
-        str.gsub("‘", "\%27")
-        str.gsub(",", "\%2C")
+        str.gsub!(" ", "%20")
+        str.gsub!("$", "\%24")
+        str.gsub!("&", "\%26")
+        str.gsub!("`", "\%60")
+        # str.gsub!(":", "\%3A")
+        str.gsub!("<", "\%3C")
+        str.gsub!(">", "\%3E")
+        str.gsub!("[", "\%5B")
+        str.gsub!("]", "\%5D")
+        str.gsub!("{", "\%7B")
+        str.gsub!("}", "\%7D")
+        str.gsub!("“", "\%22")
+        str.gsub!('"', "\%22")
+        str.gsub!("+", "\%2B")
+        str.gsub!("#", "\%23")
+        str.gsub!("\%", "\%25")
+        str.gsub!("@", "\%40")
+        # str.gsub!("/", "\%2F")
+        str.gsub!(";", "\%3B")
+        str.gsub!("=", "\%3D")
+        str.gsub!("?", "\%3F")
+        str.gsub!("\\", "\%5C")
+        str.gsub!("^", "\%5E")
+        str.gsub!("|", "\%7C")
+        str.gsub!("~", "\%7E")
+        str.gsub!("‘", "\%27")
+        str.gsub!(",", "\%2C")
         str
       end
     end

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.12"
+    VERSION = "0.1.17"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -19,18 +19,38 @@ module Pulse
 
       def extract_file_urls(response)
         return [] if response.body.nil? || response.body.empty?
+        extract_download_links(response) + extract_embedded_images(response)
+      end
 
+      def extract_download_links(response)
         parse_html(response.body)
           .css('a')
           .to_a
           .map { |link| link['href'] }
           .compact
           .select { |link| link.include? file_type }
+          .map { |link| add_base_url(link) }
+      end
+
+      def extract_embedded_images(response)
+        return [] unless scrape_images
+
+        parse_html(response.body)
+          .css('img')
+          .to_a
+          .map { |e| e["src"] }
+          .compact
+          .select { |link| link.include? file_type }
+          .map { |link| add_base_url(link) }
       end
 
       def parse_html(raw_html)
         Nokogiri::HTML(raw_html)
       end
+
+      def add_base_url(str)
+        "#{base_url}#{str}"
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.12
+  version: 0.1.17
 platform: ruby
 authors:
 - trex22
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-05-08 00:00:00.000000000 Z
+date: 2021-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty