pulse-downloader 0.1.13 → 0.1.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b348fc1621983108ea7e5bbdc2e8d590aa06706659b7017b1d72b4fde2c5dc57
-  data.tar.gz: 93ab089ee24ee1176b1bd93d7488ea764a58e505a5d4d5707f1b7dc537e403c7
+  metadata.gz: 60924c9c23911294930b5b63367a0d2f4417d437b0dcd67b384f01b470a3b689
+  data.tar.gz: 333418aa168343213ba458e5707ca767386647d3b397ed768a2fa08a7bc36b3a
 SHA512:
-  metadata.gz: 6e3a684973ec28331ef7bafcc10639ba96ba6aaa9f0ededc2dab914d9226cb323caf4afae24bdb8b23a5059b5ecb20dcfea5352dc63eb6559a1e6efb9473b867
-  data.tar.gz: 87d5bc975ca2124ba5e82d38dbc24c140e72f15e45288e7675c822e0becebeb1df26e5684fcfc96681e7c6b78f7feae3516ae77567405facfedce14db8aed78a
+  metadata.gz: 358a581f98aefc4b1a66734c9d89166b0c46b8b5d1a8686ec9925fb33b2c7ba1de8a3cf4ffac4fb1aa74597bd38876dee958391d0946b4c29345f8e2bf9a2605
+  data.tar.gz: db36a2f85b073a7fc8de3a8b92d6abf2c93ac32f70ace834f1fd1b0fb2594f348576ba562201de5df25d4597e0215046f2994af8c50412413e671b69ed230273

data/lib/pulse/downloader/client.rb

@@ -8,6 +8,7 @@ module Pulse
 
       attr_reader :url,
         :file_type,
+        :scrape_images,
         :save_data,
         :save_path,
         :read_from_save_path,
@@ -17,7 +18,8 @@ module Pulse
         :report_time,
         :start_time,
         :end_time,
-        :progress_bar
+        :progress_bar,
+        :base_url
 
       # Does not continue downloads-
       # Will only save once the file has been downloaded in memory
@@ -29,6 +31,7 @@ module Pulse
       # TODO: lib/pulse/downloader/file_downloader.rb:13: warning: URI.escape is obsolete
       def initialize(url:,
         file_type:,
+        scrape_images: false,
         save_data: false,
         save_path: '',
         read_from_save_path: false,
@@ -40,6 +43,7 @@ module Pulse
 
         @url = url
         @file_type = file_type
+        @scrape_images = scrape_images
         @save_data = save_data
         @save_path = save_path
         @read_from_save_path = read_from_save_path
@@ -48,6 +52,8 @@ module Pulse
         @save_and_dont_return = save_and_dont_return
         @report_time = report_time
         @progress_bar = progress_bar
+
+        @base_url = get_base_url
       end
 
       def call!
@@ -73,6 +79,16 @@ module Pulse
 
       private
 
+      def get_base_url
+        url_breakdown = url.split('/')
+
+        if url_breakdown.first.include?('https')
+          url_breakdown[2]
+        else
+          url_breakdown.first
+        end
+      end
+
       def get_micro_second_time
         (Time.now.to_f * 1000).to_i
       end

data/lib/pulse/downloader/file_downloader.rb

@@ -72,7 +72,7 @@ module Pulse
         str.gsub!("$", "\%24")
         str.gsub!("&", "\%26")
         str.gsub!("`", "\%60")
-        str.gsub!(":", "\%3A")
+        # str.gsub!(":", "\%3A")
         str.gsub!("<", "\%3C")
         str.gsub!(">", "\%3E")
         str.gsub!("[", "\%5B")
@@ -85,7 +85,7 @@ module Pulse
         str.gsub!("#", "\%23")
         str.gsub!("\%", "\%25")
         str.gsub!("@", "\%40")
-        str.gsub!("/", "\%2F")
+        # str.gsub!("/", "\%2F")
         str.gsub!(";", "\%3B")
         str.gsub!("=", "\%3D")
         str.gsub!("?", "\%3F")

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.13"
+    VERSION = "0.1.18"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -19,18 +19,42 @@ module Pulse
 
       def extract_file_urls(response)
         return [] if response.body.nil? || response.body.empty?
+        extract_download_links(response) + extract_embedded_images(response)
+      end
 
+      def extract_download_links(response)
         parse_html(response.body)
           .css('a')
           .to_a
           .map { |link| link['href'] }
           .compact
           .select { |link| link.include? file_type }
+          .map { |link| add_base_url(link) }
+      end
+
+      def extract_embedded_images(response)
+        return [] unless scrape_images
+
+        parse_html(response.body)
+          .css('img')
+          .to_a
+          .map { |e| e["src"] }
+          .compact
+          .select { |link| link.include? file_type }
+          .map { |link| add_base_url(link) }
       end
 
       def parse_html(raw_html)
         Nokogiri::HTML(raw_html)
       end
+
+      def add_base_url(str)
+        unless str.include?(base_url)
+          "#{base_url}#{str}"
+        else
+          str
+        end
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.13
+  version: 0.1.18
 platform: ruby
 authors:
 - trex22
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-05-08 00:00:00.000000000 Z
+date: 2021-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty