pulse-downloader 0.1.19 → 0.1.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4ce4c8fd47e92ddc487cd80c36564a21357a3ad79deee82d0af70812c935ab1e
-  data.tar.gz: 3b2dbfc9387fe3c496ae112d3d8c6a7af6cf1b3b0c550fed0da358d9bf798efa
+  metadata.gz: 13ed97a9caedc58bca6c4247f2d56d7456aecafe0f2eecb65b6c25b5de404fd4
+  data.tar.gz: 0d450684690c49812a63a0aec81f9e9d77e6074898a6ac2be9b7143fe5989adb
 SHA512:
-  metadata.gz: 1b72add3be0201cfc9c395d790c26c77fc8b8f3383048fc334d8b84695d1d0fd8d9dca17af16242a1ff74c54b8a37e81e40262b90c293507b4668902e23477a5
-  data.tar.gz: b4a2a5087afe6d10ef1c424169862f146f9911d8dd13369ffe86318bab3a083c815a65449375f3a47d1f9864a6de23b5db9417e509285d8647a47637220b9cdb
+  metadata.gz: '001790c43ba48c68ad1222d70f1352e5b399384fb7be69c6951ac80fd629ed2009fcc4a1292e23ab712ddaa1a9f206a59b1abd7f965625486160224e4e7a77bb'
+  data.tar.gz: d49f98068eec42477b7069948f4a7c72b37b4e166c6f843b4e7e81763c27eb5e5561115f7b15800e48ed2cd86f5397e6a21f7d3eaf2816a91b017795399fbb9f

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.19"
+    VERSION = "0.1.24"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -1,7 +1,7 @@
 module Pulse
   module Downloader
     module WebPageParser
-      def fetch_file_paths
+      def fetch_file_paths(custom_path_root=nil)
         @start_time = get_micro_second_time
 
         response = HTTParty.get(url, verify: verify_ssl)
@@ -12,29 +12,36 @@ module Pulse
           print_time
         end
 
-        extract_file_urls(response)
+        if file_type.is_a?(Array)
+          file_type.flat_map do |type|
+            extract_file_urls(response, custom_path_root, type)
+          end
+        else
+          extract_file_urls(response, custom_path_root, file_type)
+        end
       end
 
       private
 
-      def extract_file_urls(response)
+      def extract_file_urls(response, custom_path_root, type)
         return [] if response.body.nil? || response.body.empty?
         (
-          extract_download_links(response) + extract_embedded_images(response)
+          extract_download_links(response, custom_path_root, type) +
+            extract_embedded_images(response, custom_path_root, type)
         ).uniq
       end
 
-      def extract_download_links(response)
+      def extract_download_links(response, custom_path_root, type)
         parse_html(response.body)
           .css('a')
           .to_a
           .map { |link| link['href'] }
           .compact
-          .select { |link| link.include? file_type }
+          .select { |link| (link.include? type || link.include?(custom_path_root)) }
           .map { |link| add_base_url(link) }
       end
 
-      def extract_embedded_images(response)
+      def extract_embedded_images(response, custom_path_root, type)
         return [] unless scrape_images
 
         parse_html(response.body)
@@ -42,7 +49,7 @@ module Pulse
           .to_a
           .map { |e| e["src"] }
           .compact
-          .select { |link| link.include? file_type }
+          .select { |link| (link.include? type || link.include?(custom_path_root)) }
           .map { |link| add_base_url(link) }
       end
 
@@ -51,7 +58,7 @@ module Pulse
       end
 
       def add_base_url(str)
-        unless str.include?(base_url)
+        if !str.include?('https://') && !str.include?(base_url)
           "https://#{base_url}#{str}"
         else
           str

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.19
+  version: 0.1.24
 platform: ruby
 authors:
 - trex22