pulse-downloader 0.1.30 → 0.1.34

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5b1f82d865e48f194bbbbad697b5049c0a40ede8b74442032dfe6812b75e7ad9
-  data.tar.gz: e8f100c954fc76f4f6f32a262ebe353b9a68b7e468c0269cb0ee718f91a7d18c
+  metadata.gz: ee4cd8b3e10c09d8a52ddf222d3d423ffb188a0b7eb4993febbcc97b74644764
+  data.tar.gz: 2126a1a1cecc91c2b0aa275ca17322deac30779682acbd6c08d4dad9f48d412e
 SHA512:
-  metadata.gz: 8e1ce2a8e64b23bf74c36c39a2d9112940eab4778da968582b33dc3818c8803ff8d4a4c6e849cc741cc4f6d6ddf4a778e537173a8c5cfaa09ad51baf18549e86
-  data.tar.gz: f996baddaf407802ef415dbc8177c08d8845d007eaf24e9e67ca0aaa8fed770ac1c4c71deaf4632998cc0e8890e23dac4363e9ab77054b670b2cb071db4d2065
+  metadata.gz: 0b38229caa3120433d2434acc5ca8122c86fef5957db323895ea781473ef1cf40a9ab202f581c48de5ab27f438bfd0d046248867e5ea924ff9d473828d84763f
+  data.tar.gz: fd5908c8b30f58f07fa6496db39b88ba24a9f1dde4a6d87df5e65ba133b00241e6e836aeb4d66badbe832b6c4eb8621f98c1cbca03d3e94cfc2c7cbc18bef2d6

data/Gemfile

@@ -2,6 +2,3 @@ source "https://rubygems.org"
 
 # Specify your gem's dependencies in pulse-downloader.gemspec
 gemspec
-
-gem "rake", "~> 12.0"
-gem "minitest", "~> 5.0"

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pulse-downloader (0.1.26)
+    pulse-downloader (0.1.31)
       active_attr (~> 0.15)
       httparty (~> 0.18)
       nokogiri (~> 1.11)
@@ -10,15 +10,15 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (6.1.3.2)
-      actionview (= 6.1.3.2)
-      activesupport (= 6.1.3.2)
+    actionpack (6.1.4.1)
+      actionview (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
       rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actionview (6.1.3.2)
-      activesupport (= 6.1.3.2)
+    actionview (6.1.4.1)
+      activesupport (= 6.1.4.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
@@ -27,20 +27,20 @@ GEM
       actionpack (>= 3.0.2, < 7.0)
       activemodel (>= 3.0.2, < 7.0)
       activesupport (>= 3.0.2, < 7.0)
-    activemodel (6.1.3.2)
-      activesupport (= 6.1.3.2)
-    activesupport (6.1.3.2)
+    activemodel (6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activesupport (6.1.4.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
       zeitwerk (~> 2.3)
-    addressable (2.7.0)
+    addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     ansi (1.5.0)
     builder (3.2.4)
     coderay (1.1.3)
-    concurrent-ruby (1.1.8)
+    concurrent-ruby (1.1.9)
     crack (0.4.5)
       rexml
     crass (1.0.6)
@@ -52,13 +52,13 @@ GEM
       multi_xml (>= 0.5.2)
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
-    loofah (2.9.1)
+    loofah (2.12.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2021.0225)
+    mime-types-data (3.2021.0901)
     minitest (5.14.4)
     minitest-focus (1.1.2)
       minitest (>= 4, < 6)
@@ -69,7 +69,7 @@ GEM
       ruby-progressbar
     mocha (1.11.2)
     multi_xml (0.6.0)
-    nokogiri (1.11.5-x86_64-linux)
+    nokogiri (1.12.4-x86_64-linux)
       racc (~> 1.4)
     options (2.3.2)
     progress_bar (1.3.3)
@@ -86,9 +86,9 @@ GEM
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.3.0)
+    rails-html-sanitizer (1.4.2)
       loofah (~> 2.3)
-    rake (12.3.3)
+    rake (13.0.6)
     rexml (3.2.5)
     ruby-progressbar (1.11.0)
     timecop (0.9.4)
@@ -111,7 +111,7 @@ DEPENDENCIES
   mocha (~> 1.11.2)
   pry (~> 0.13)
   pulse-downloader!
-  rake (~> 12.0)
+  rake (~> 13.0)
   timecop (~> 0.9.1)
   webmock (~> 3.8.3)
 

data/README.md

@@ -29,6 +29,7 @@ client = Pulse::Downloader::Client.new(
   save_data: true,
   save_path: '',
   read_from_save_path: false,
+  traverse_folders: false,
   verify_ssl: true,
   drop_exitsing_files_in_path: false,
   save_and_dont_return: true,
@@ -38,6 +39,8 @@ client = Pulse::Downloader::Client.new(
 )
 
 client.call!
+
+client.file_paths # Will give you the list of file paths
 ```
 
 ## Development

data/lib/pulse/downloader/client.rb

@@ -12,6 +12,7 @@ module Pulse
         :save_data,
         :save_path,
         :read_from_save_path,
+        :traverse_folders,
         :verify_ssl,
         :headers,
         :drop_exitsing_files_in_path,
@@ -20,7 +21,8 @@ module Pulse
         :start_time,
         :end_time,
         :progress_bar,
-        :base_url
+        :base_url,
+        :file_paths
 
       # Does not continue downloads-
       # Will only save once the file has been downloaded in memory
@@ -28,7 +30,6 @@ module Pulse
       # TODO: Validation
       # TODO: Retry
       # TODO: DNS
-      # TODO: Multiple filetypes
       # TODO: lib/pulse/downloader/file_downloader.rb:13: warning: URI.escape is obsolete
       def initialize(url:,
         file_type:,
@@ -36,6 +37,7 @@ module Pulse
         save_data: false,
         save_path: '',
         read_from_save_path: false,
+        traverse_folders: false,
         verify_ssl: true,
         headers: nil,
         drop_exitsing_files_in_path: false,
@@ -49,6 +51,7 @@ module Pulse
         @save_data = save_data
         @save_path = save_path
         @read_from_save_path = read_from_save_path
+        @traverse_folders = traverse_folders
         @verify_ssl = verify_ssl
         @headers = headers
         @drop_exitsing_files_in_path = drop_exitsing_files_in_path
@@ -66,12 +69,14 @@ module Pulse
       def call
         return false unless valid?
 
+        @file_paths = fetch_file_paths
+
         if @progress_bar
-          @progress_bar = ::ProgressBar.new(fetch_file_paths.size)
+          @progress_bar = ::ProgressBar.new(file_paths.size)
         end
 
-        fetch_file_paths.map do |file_path|
-          download(file_path, @progress_bar)
+        file_paths.map do |file_path|
+          download(file_path, @progress_bar) if save_data
           @progress_bar.increment!
         end
       end

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.30"
+    VERSION = "0.1.34"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -2,9 +2,25 @@ module Pulse
   module Downloader
     module WebPageParser
       def fetch_file_paths(custom_path_root=nil)
+        if traverse_folders
+          fetch_folders(url).each do |folder_url|
+            fetch_and_parse_response(folder_url, custom_path_root)
+          end
+        else
+          fetch_and_parse_response(url, custom_path_root)
+        end
+      end
+
+      private
+
+      def fetch_folders(base_url)
+
+      end
+
+      def fetch_and_parse_response(folder_url, custom_path_root)
         @start_time = get_micro_second_time
 
-        response = HTTParty.get(url, verify: verify_ssl, headers: headers)
+        response = HTTParty.get(folder_url, verify: verify_ssl, headers: headers)
 
         @end_time = get_micro_second_time
 
@@ -12,6 +28,10 @@ module Pulse
           print_time
         end
 
+        parse_response(response, custom_path_root, file_type)
+      end
+
+      def parse_response(response, custom_path_root, file_type)
         if file_type.is_a?(Array)
           file_type.flat_map do |type|
             extract_file_urls(response, custom_path_root, type)
@@ -21,8 +41,6 @@ module Pulse
         end
       end
 
-      private
-
       def extract_file_urls(response, custom_path_root, type)
         return [] if response.body.nil? || response.body.empty?
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.30
+  version: 0.1.34
 platform: ruby
 authors:
 - trex22
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-05-24 00:00:00.000000000 Z
+date: 2021-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty