pulse-downloader 0.1.18 → 0.1.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 60924c9c23911294930b5b63367a0d2f4417d437b0dcd67b384f01b470a3b689
-  data.tar.gz: 333418aa168343213ba458e5707ca767386647d3b397ed768a2fa08a7bc36b3a
+  metadata.gz: e354feaf3ad68d91b2b58b1410ad297582ace9b6f3493b17cf8ae11d11c18d8d
+  data.tar.gz: 3e036681abb31307b958bba6038d1fda3a9634195d71c61af3e8896bd695b22a
 SHA512:
-  metadata.gz: 358a581f98aefc4b1a66734c9d89166b0c46b8b5d1a8686ec9925fb33b2c7ba1de8a3cf4ffac4fb1aa74597bd38876dee958391d0946b4c29345f8e2bf9a2605
-  data.tar.gz: db36a2f85b073a7fc8de3a8b92d6abf2c93ac32f70ace834f1fd1b0fb2594f348576ba562201de5df25d4597e0215046f2994af8c50412413e671b69ed230273
+  metadata.gz: cf401b59b944a0bdca308998c19f4c8b996fadf60cc8118c06a909b5eaa1ca98d9f0f3d234af7050ce71611475d8f9a9c4a79503003649870a5d7f18dc62caa0
+  data.tar.gz: 98bd01ea727425dad6a932fd431c241b023d954134827ac781faab9b90559a3d05c75916eb4e0bf99d4f8f4c4244b2ce7b019df8ced9c474ad8efe18fa4e7e7d

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.18"
+    VERSION = "0.1.23"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -1,7 +1,7 @@
 module Pulse
   module Downloader
     module WebPageParser
-      def fetch_file_paths
+      def fetch_file_paths(custom_path_root=nil)
         @start_time = get_micro_second_time
 
         response = HTTParty.get(url, verify: verify_ssl)
@@ -12,27 +12,30 @@ module Pulse
           print_time
         end
 
-        extract_file_urls(response)
+        extract_file_urls(response, custom_path_root)
       end
 
       private
 
-      def extract_file_urls(response)
+      def extract_file_urls(response, custom_path_root)
         return [] if response.body.nil? || response.body.empty?
-        extract_download_links(response) + extract_embedded_images(response)
+        (
+          extract_download_links(response, custom_path_root) +
+            extract_embedded_images(response, custom_path_root)
+        ).uniq
       end
 
-      def extract_download_links(response)
+      def extract_download_links(response, custom_path_root)
         parse_html(response.body)
           .css('a')
           .to_a
           .map { |link| link['href'] }
           .compact
-          .select { |link| link.include? file_type }
+          .select { |link| (link.include? file_type || link.include?(custom_path_root)) }
           .map { |link| add_base_url(link) }
       end
 
-      def extract_embedded_images(response)
+      def extract_embedded_images(response, custom_path_root)
         return [] unless scrape_images
 
         parse_html(response.body)
@@ -40,7 +43,7 @@ module Pulse
           .to_a
           .map { |e| e["src"] }
           .compact
-          .select { |link| link.include? file_type }
+          .select { |link| (link.include? file_type || link.include?(custom_path_root)) }
           .map { |link| add_base_url(link) }
       end
 
@@ -49,8 +52,8 @@ module Pulse
       end
 
       def add_base_url(str)
-        unless str.include?(base_url)
-          "#{base_url}#{str}"
+        if !str.include?('https://') && !str.include?(base_url)
+          "https://#{base_url}#{str}"
         else
           str
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.18
+  version: 0.1.23
 platform: ruby
 authors:
 - trex22