pulse-downloader 0.1.34 → 0.1.35

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee4cd8b3e10c09d8a52ddf222d3d423ffb188a0b7eb4993febbcc97b74644764
-  data.tar.gz: 2126a1a1cecc91c2b0aa275ca17322deac30779682acbd6c08d4dad9f48d412e
+  metadata.gz: df940062ab2c0dcdc96b256b76e8198683776d7bdb834198c0f75da85d5e63fb
+  data.tar.gz: 158f5746ceb0d820934e126cae7ca53dfca0a841ae8a588b2b955c85de5abc7f
 SHA512:
-  metadata.gz: 0b38229caa3120433d2434acc5ca8122c86fef5957db323895ea781473ef1cf40a9ab202f581c48de5ab27f438bfd0d046248867e5ea924ff9d473828d84763f
-  data.tar.gz: fd5908c8b30f58f07fa6496db39b88ba24a9f1dde4a6d87df5e65ba133b00241e6e836aeb4d66badbe832b6c4eb8621f98c1cbca03d3e94cfc2c7cbc18bef2d6
+  metadata.gz: 2494b72940a92e4b13fc44c8c9c81ff63e48a35940e9bcc6437235afdd4c1ca2fdc43098b71f976ce42d5fa65b53979aa3b055b511c91c1ca6b0ff64a0dd472c
+  data.tar.gz: 1c40920e57fdbb4184034cc91953f82c5babf1c09146027b9b887b1bac5d8b45f4e386b21133bd44625f8adb8843daac8b4652e35e94ebf25188738d02d8a36c

data/lib/pulse/downloader/client.rb

@@ -22,7 +22,8 @@ module Pulse
         :end_time,
         :progress_bar,
         :base_url,
-        :file_paths
+        :file_paths,
+        :folder_urls
 
       # Does not continue downloads-
       # Will only save once the file has been downloaded in memory
@@ -60,6 +61,7 @@ module Pulse
         @progress_bar = progress_bar
 
         @base_url = get_base_url
+        @folder_urls = []
       end
 
       def call!

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.34"
+    VERSION = "0.1.35"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -14,10 +14,23 @@ module Pulse
       private
 
       def fetch_folders(base_url)
+        current_paths = extract_hrefs(get_response(folder_url), custom_path_root)
+        return unless current_paths.compact.size > 0
 
+        @folder_urls = folder_urls.union(current_paths).uniq.compact
+
+        current_paths.each do |path|
+          fetch_folders(path)
+        end
+
+        folder_urls
       end
 
       def fetch_and_parse_response(folder_url, custom_path_root)
+        parse_response(get_response(folder_url), custom_path_root, file_type)
+      end
+
+      def get_response(folder_url)
         @start_time = get_micro_second_time
 
         response = HTTParty.get(folder_url, verify: verify_ssl, headers: headers)
@@ -28,7 +41,7 @@ module Pulse
           print_time
         end
 
-        parse_response(response, custom_path_root, file_type)
+        response
       end
 
       def parse_response(response, custom_path_root, file_type)
@@ -51,6 +64,14 @@ module Pulse
         ).uniq
       end
 
+      def extract_hrefs(response, custom_path_root)
+        parse_html(response.body)
+          .css('a')
+          .map { |link| link['href'] }
+          .reject { |link| link == "../" }
+          .map { |link| add_base_url(link, custom_path_root) }
+      end
+
       def extract_all_urls(response, custom_path_root, type)
         parse_html(response.body)
           .to_s

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.34
+  version: 0.1.35
 platform: ruby
 authors:
 - trex22