pulse-downloader 0.1.34 → 0.1.38

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee4cd8b3e10c09d8a52ddf222d3d423ffb188a0b7eb4993febbcc97b74644764
-  data.tar.gz: 2126a1a1cecc91c2b0aa275ca17322deac30779682acbd6c08d4dad9f48d412e
+  metadata.gz: 15b3ba4650d2aa0c01ba6f5456888889bfd888405177aa956f092ba2c0691065
+  data.tar.gz: d28d3d28516d5cbdeaa255923e79e55526a364f70389a3d716ce9e8dec90af08
 SHA512:
-  metadata.gz: 0b38229caa3120433d2434acc5ca8122c86fef5957db323895ea781473ef1cf40a9ab202f581c48de5ab27f438bfd0d046248867e5ea924ff9d473828d84763f
-  data.tar.gz: fd5908c8b30f58f07fa6496db39b88ba24a9f1dde4a6d87df5e65ba133b00241e6e836aeb4d66badbe832b6c4eb8621f98c1cbca03d3e94cfc2c7cbc18bef2d6
+  metadata.gz: ff848ed64e1adefe773378d9f8a6ccb2d9b7f533a58a10e008b8f7d1b1d678c16978426c0c4a3de8aaa2797c1647304b7dd996fcb979a4cc7a831ce229163fc1
+  data.tar.gz: d9f01143d5d1a27ba38516b0ebed0a4ab49078e796f513d8351a90bbc8506077aa375b8b1a2ee8576b81836957f90ae31b5db31e8fdcc4b18a3dc30170f9b182

data/lib/pulse/downloader/client.rb

@@ -22,7 +22,8 @@ module Pulse
         :end_time,
         :progress_bar,
         :base_url,
-        :file_paths
+        :file_paths,
+        :folder_urls
 
       # Does not continue downloads-
       # Will only save once the file has been downloaded in memory
@@ -60,6 +61,7 @@ module Pulse
         @progress_bar = progress_bar
 
         @base_url = get_base_url
+        @folder_urls = []
       end
 
       def call!

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.34"
+    VERSION = "0.1.38"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -3,7 +3,7 @@ module Pulse
     module WebPageParser
       def fetch_file_paths(custom_path_root=nil)
         if traverse_folders
-          fetch_folders(url).each do |folder_url|
+          fetch_folders(url, custom_path_root).each do |folder_url|
             fetch_and_parse_response(folder_url, custom_path_root)
           end
         else
@@ -13,11 +13,24 @@ module Pulse
 
       private
 
-      def fetch_folders(base_url)
+      def fetch_folders(folder_url, custom_path_root)
+        current_paths = extract_hrefs(get_response(folder_url), custom_path_root)
+        return unless current_paths.compact.size > 0
 
+        @folder_urls = folder_urls.union(current_paths).uniq.compact
+
+        current_paths.each do |path|
+          fetch_folders(path, custom_path_root)
+        end
+
+        folder_urls
       end
 
       def fetch_and_parse_response(folder_url, custom_path_root)
+        parse_response(get_response(folder_url), custom_path_root, file_type)
+      end
+
+      def get_response(folder_url)
         @start_time = get_micro_second_time
 
         response = HTTParty.get(folder_url, verify: verify_ssl, headers: headers)
@@ -28,7 +41,7 @@ module Pulse
           print_time
         end
 
-        parse_response(response, custom_path_root, file_type)
+        response
       end
 
       def parse_response(response, custom_path_root, file_type)
@@ -51,6 +64,14 @@ module Pulse
         ).uniq
       end
 
+      def extract_hrefs(response, custom_path_root)
+        parse_html(response.body)
+          .css('a')
+          .map { |link| "/#{link['href']}" }
+          .reject { |link| link == "../" || link == "/../" }
+          .map { |link| add_base_url(link, custom_path_root) }
+      end
+
       def extract_all_urls(response, custom_path_root, type)
         parse_html(response.body)
           .to_s

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.34
+  version: 0.1.38
 platform: ruby
 authors:
 - trex22