pulse-downloader 0.1.20 → 0.1.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8a2d2732c3a0307e7dffaf1ec3b60691e7b54ca3de4c4084325908f6fdf1b801
-  data.tar.gz: 06b3a802a55f186f8fccceec0e29d6b6e3dbade7be776699a6a327ccacf68060
+  metadata.gz: 48153054f979c1f1ed2e36c211edb598f921b41e6f5e0b71bfd8368eaa4112f3
+  data.tar.gz: 1c831b86a9ead79b51446ed0b911ee54d3b410437d20214fddb57da808a3938a
 SHA512:
-  metadata.gz: 27c5f745685fbb5b2c207113b4fe28eb97b8638551afb931e3be1e4d9f073c5959ba0cc1ed59db44fbdfaae26218840ca8663da95fba37ef7c0cf18a37f2aefc
-  data.tar.gz: 3a06bbe0fafe7329eec9ba1d90e483da181f309915b06bfb67d49c3de9f1c96402047d775f672ed172979ca210d988e7063acd5c53860dc61bfcf508a6f64457
+  metadata.gz: f0a0e5d9dbeca80d2de42ae1e655191e9aa01c557992b141412cd1dfe86751b3e047b951fc2f6bbf28ea487b7bc8f80d1d173d76c129ca38194cda34f894ef7a
+  data.tar.gz: cf2a9d907e5f35a792c4d9b7f89ea0e0a96b50de499ade69b62afe0108857a7324836021b3ba338d22c05744789b8e252db08a1ae36177f3a5f671003bee6a32

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.20"
+    VERSION = "0.1.25"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -1,7 +1,7 @@
 module Pulse
   module Downloader
     module WebPageParser
-      def fetch_file_paths
+      def fetch_file_paths(custom_path_root=nil)
         @start_time = get_micro_second_time
 
         response = HTTParty.get(url, verify: verify_ssl)
@@ -12,29 +12,47 @@ module Pulse
           print_time
         end
 
-        extract_file_urls(response)
+        if file_type.is_a?(Array)
+          file_type.flat_map do |type|
+            extract_file_urls(response, custom_path_root, type)
+          end
+        else
+          extract_file_urls(response, custom_path_root, file_type)
+        end
       end
 
       private
 
-      def extract_file_urls(response)
+      def extract_file_urls(response, custom_path_root, type)
         return [] if response.body.nil? || response.body.empty?
         (
-          extract_download_links(response) + extract_embedded_images(response)
+          extract_all_urls(response, custom_path_root, type) +
+            extract_download_links(response, custom_path_root, type) +
+            extract_embedded_images(response, custom_path_root, type)
         ).uniq
       end
 
-      def extract_download_links(response)
+      def extract_all_urls(response, custom_path_root, type)
+        parse_html(response.body)
+          .to_s
+          .split(/\s+/)
+          .find_all { |u| u =~ /^https?:/ }
+          .compact
+          .select { |link| (link.include? type || link.include?(custom_path_root)) }
+          .map { |link| add_base_url(link) }
+      end
+
+      def extract_download_links(response, custom_path_root, type)
         parse_html(response.body)
           .css('a')
           .to_a
           .map { |link| link['href'] }
           .compact
-          .select { |link| link.include? file_type }
+          .select { |link| (link.include? type || link.include?(custom_path_root)) }
           .map { |link| add_base_url(link) }
       end
 
-      def extract_embedded_images(response)
+      def extract_embedded_images(response, custom_path_root, type)
         return [] unless scrape_images
 
         parse_html(response.body)
@@ -42,7 +60,7 @@ module Pulse
           .to_a
           .map { |e| e["src"] }
           .compact
-          .select { |link| link.include? file_type }
+          .select { |link| (link.include? type || link.include?(custom_path_root)) }
           .map { |link| add_base_url(link) }
       end
 
@@ -51,9 +69,7 @@ module Pulse
       end
 
       def add_base_url(str)
-        unless str.include?('https://')
-          str
-        unless str.include?(base_url)
+        if !str.include?('https://') && !str.include?(base_url)
           "https://#{base_url}#{str}"
         else
           str

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.20
+  version: 0.1.25
 platform: ruby
 authors:
 - trex22