pulse-downloader 0.1.31 → 0.1.35

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3036b6512e69bffe3a1c2b4dbfb156c4b932910dbf95ca4d61d7e89307a86de5
-  data.tar.gz: 2aa3cd36ea992d59463d3127ec589eb848b1e59258e1714b9673d2dbd93c7e8d
+  metadata.gz: df940062ab2c0dcdc96b256b76e8198683776d7bdb834198c0f75da85d5e63fb
+  data.tar.gz: 158f5746ceb0d820934e126cae7ca53dfca0a841ae8a588b2b955c85de5abc7f
 SHA512:
-  metadata.gz: 9df1e6f4b4136a6061e6222df80c2899b28b73f7b68648c7c2ecb7007eb8aa6c26c4a18fd6d65f9c11651504068ef053d9007614dd6fc09765ccdcae2b9946e9
-  data.tar.gz: 5ac6bfb3d0bed3bddbefdd752418b6ae6f86c50aa76dfff4dbc1806fcfc0f2336215646af825c787d5cd3cd6bc08b1fd9ec0c5747f50cefa0ed5da449bf7f7e3
+  metadata.gz: 2494b72940a92e4b13fc44c8c9c81ff63e48a35940e9bcc6437235afdd4c1ca2fdc43098b71f976ce42d5fa65b53979aa3b055b511c91c1ca6b0ff64a0dd472c
+  data.tar.gz: 1c40920e57fdbb4184034cc91953f82c5babf1c09146027b9b887b1bac5d8b45f4e386b21133bd44625f8adb8843daac8b4652e35e94ebf25188738d02d8a36c

data/README.md

@@ -29,6 +29,7 @@ client = Pulse::Downloader::Client.new(
   save_data: true,
   save_path: '',
   read_from_save_path: false,
+  traverse_folders: false,
   verify_ssl: true,
   drop_exitsing_files_in_path: false,
   save_and_dont_return: true,
@@ -38,6 +39,8 @@ client = Pulse::Downloader::Client.new(
 )
 
 client.call!
+
+client.file_paths # Will give you the list of file paths
 ```
 
 ## Development

data/lib/pulse/downloader/client.rb

@@ -12,6 +12,7 @@ module Pulse
         :save_data,
         :save_path,
         :read_from_save_path,
+        :traverse_folders,
         :verify_ssl,
         :headers,
         :drop_exitsing_files_in_path,
@@ -20,7 +21,9 @@ module Pulse
         :start_time,
         :end_time,
         :progress_bar,
-        :base_url
+        :base_url,
+        :file_paths,
+        :folder_urls
 
       # Does not continue downloads-
       # Will only save once the file has been downloaded in memory
@@ -28,7 +31,6 @@ module Pulse
       # TODO: Validation
       # TODO: Retry
       # TODO: DNS
-      # TODO: Multiple filetypes
       # TODO: lib/pulse/downloader/file_downloader.rb:13: warning: URI.escape is obsolete
       def initialize(url:,
         file_type:,
@@ -36,6 +38,7 @@ module Pulse
         save_data: false,
         save_path: '',
         read_from_save_path: false,
+        traverse_folders: false,
         verify_ssl: true,
         headers: nil,
         drop_exitsing_files_in_path: false,
@@ -49,6 +52,7 @@ module Pulse
         @save_data = save_data
         @save_path = save_path
         @read_from_save_path = read_from_save_path
+        @traverse_folders = traverse_folders
         @verify_ssl = verify_ssl
         @headers = headers
         @drop_exitsing_files_in_path = drop_exitsing_files_in_path
@@ -57,6 +61,7 @@ module Pulse
         @progress_bar = progress_bar
 
         @base_url = get_base_url
+        @folder_urls = []
       end
 
       def call!
@@ -66,12 +71,14 @@ module Pulse
       def call
         return false unless valid?
 
+        @file_paths = fetch_file_paths
+
         if @progress_bar
-          @progress_bar = ::ProgressBar.new(fetch_file_paths.size)
+          @progress_bar = ::ProgressBar.new(file_paths.size)
         end
 
-        fetch_file_paths.map do |file_path|
-          download(file_path, @progress_bar)
+        file_paths.map do |file_path|
+          download(file_path, @progress_bar) if save_data
           @progress_bar.increment!
         end
       end

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.31"
+    VERSION = "0.1.35"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -2,9 +2,38 @@ module Pulse
   module Downloader
     module WebPageParser
       def fetch_file_paths(custom_path_root=nil)
+        if traverse_folders
+          fetch_folders(url).each do |folder_url|
+            fetch_and_parse_response(folder_url, custom_path_root)
+          end
+        else
+          fetch_and_parse_response(url, custom_path_root)
+        end
+      end
+
+      private
+
+      def fetch_folders(base_url)
+        current_paths = extract_hrefs(get_response(folder_url), custom_path_root)
+        return unless current_paths.compact.size > 0
+
+        @folder_urls = folder_urls.union(current_paths).uniq.compact
+
+        current_paths.each do |path|
+          fetch_folders(path)
+        end
+
+        folder_urls
+      end
+
+      def fetch_and_parse_response(folder_url, custom_path_root)
+        parse_response(get_response(folder_url), custom_path_root, file_type)
+      end
+
+      def get_response(folder_url)
         @start_time = get_micro_second_time
 
-        response = HTTParty.get(url, verify: verify_ssl, headers: headers)
+        response = HTTParty.get(folder_url, verify: verify_ssl, headers: headers)
 
         @end_time = get_micro_second_time
 
@@ -12,6 +41,10 @@ module Pulse
           print_time
         end
 
+        response
+      end
+
+      def parse_response(response, custom_path_root, file_type)
         if file_type.is_a?(Array)
           file_type.flat_map do |type|
             extract_file_urls(response, custom_path_root, type)
@@ -21,8 +54,6 @@ module Pulse
         end
       end
 
-      private
-
       def extract_file_urls(response, custom_path_root, type)
         return [] if response.body.nil? || response.body.empty?
 
@@ -33,6 +64,14 @@ module Pulse
         ).uniq
       end
 
+      def extract_hrefs(response, custom_path_root)
+        parse_html(response.body)
+          .css('a')
+          .map { |link| link['href'] }
+          .reject { |link| link == "../" }
+          .map { |link| add_base_url(link, custom_path_root) }
+      end
+
       def extract_all_urls(response, custom_path_root, type)
         parse_html(response.body)
           .to_s

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.31
+  version: 0.1.35
 platform: ruby
 authors:
 - trex22