pulse-downloader 0.1.33 → 0.1.34

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea8f7f1cba1f575af4fb82c8d83d7108decbdc8eabf38be38693b2f94c5d4b95
-  data.tar.gz: 69deb413bdce72404ca9a76e5d5d01014214559c71a1aad421c8b3e32c400f52
+  metadata.gz: ee4cd8b3e10c09d8a52ddf222d3d423ffb188a0b7eb4993febbcc97b74644764
+  data.tar.gz: 2126a1a1cecc91c2b0aa275ca17322deac30779682acbd6c08d4dad9f48d412e
 SHA512:
-  metadata.gz: 1f1633832d05cf665d116cd9180c66b02d7c0a0c5dbf77db03d758fa9f0f051982761125cbdfc4914e3dd53b5ee6c91fc9b74fe16cd9c992098e7f72df78ac52
-  data.tar.gz: 4f571acd0d6019506a9cb58b5e86bcb8c79cbc856406342a1325864ee6f771b7c3bdc09e65d8bba6ee79d1d77b59f4a37b5e771e6b8b51d959350d32142bf378
+  metadata.gz: 0b38229caa3120433d2434acc5ca8122c86fef5957db323895ea781473ef1cf40a9ab202f581c48de5ab27f438bfd0d046248867e5ea924ff9d473828d84763f
+  data.tar.gz: fd5908c8b30f58f07fa6496db39b88ba24a9f1dde4a6d87df5e65ba133b00241e6e836aeb4d66badbe832b6c4eb8621f98c1cbca03d3e94cfc2c7cbc18bef2d6

data/README.md

@@ -29,6 +29,7 @@ client = Pulse::Downloader::Client.new(
   save_data: true,
   save_path: '',
   read_from_save_path: false,
+  traverse_folders: false,
   verify_ssl: true,
   drop_exitsing_files_in_path: false,
   save_and_dont_return: true,
@@ -38,6 +39,8 @@ client = Pulse::Downloader::Client.new(
 )
 
 client.call!
+
+client.file_paths # Will give you the list of file paths
 ```
 
 ## Development

data/lib/pulse/downloader/client.rb

@@ -12,6 +12,7 @@ module Pulse
         :save_data,
         :save_path,
         :read_from_save_path,
+        :traverse_folders,
         :verify_ssl,
         :headers,
         :drop_exitsing_files_in_path,
@@ -29,7 +30,6 @@ module Pulse
       # TODO: Validation
       # TODO: Retry
       # TODO: DNS
-      # TODO: Multiple filetypes
       # TODO: lib/pulse/downloader/file_downloader.rb:13: warning: URI.escape is obsolete
       def initialize(url:,
         file_type:,
@@ -37,6 +37,7 @@ module Pulse
         save_data: false,
         save_path: '',
         read_from_save_path: false,
+        traverse_folders: false,
         verify_ssl: true,
         headers: nil,
         drop_exitsing_files_in_path: false,
@@ -50,6 +51,7 @@ module Pulse
         @save_data = save_data
         @save_path = save_path
         @read_from_save_path = read_from_save_path
+        @traverse_folders = traverse_folders
         @verify_ssl = verify_ssl
         @headers = headers
         @drop_exitsing_files_in_path = drop_exitsing_files_in_path

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.33"
+    VERSION = "0.1.34"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -2,9 +2,25 @@ module Pulse
   module Downloader
     module WebPageParser
       def fetch_file_paths(custom_path_root=nil)
+        if traverse_folders
+          fetch_folders(url).each do |folder_url|
+            fetch_and_parse_response(folder_url, custom_path_root)
+          end
+        else
+          fetch_and_parse_response(url, custom_path_root)
+        end
+      end
+
+      private
+
+      def fetch_folders(base_url)
+
+      end
+
+      def fetch_and_parse_response(folder_url, custom_path_root)
         @start_time = get_micro_second_time
 
-        response = HTTParty.get(url, verify: verify_ssl, headers: headers)
+        response = HTTParty.get(folder_url, verify: verify_ssl, headers: headers)
 
         @end_time = get_micro_second_time
 
@@ -12,6 +28,10 @@ module Pulse
           print_time
         end
 
+        parse_response(response, custom_path_root, file_type)
+      end
+
+      def parse_response(response, custom_path_root, file_type)
         if file_type.is_a?(Array)
           file_type.flat_map do |type|
             extract_file_urls(response, custom_path_root, type)
@@ -21,8 +41,6 @@ module Pulse
         end
       end
 
-      private
-
       def extract_file_urls(response, custom_path_root, type)
         return [] if response.body.nil? || response.body.empty?
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.33
+  version: 0.1.34
 platform: ruby
 authors:
 - trex22