pulse-downloader 0.1.25 → 0.1.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 48153054f979c1f1ed2e36c211edb598f921b41e6f5e0b71bfd8368eaa4112f3
-  data.tar.gz: 1c831b86a9ead79b51446ed0b911ee54d3b410437d20214fddb57da808a3938a
+  metadata.gz: b59437d8b4d75cbb7c68010644dbb046f6860e9c37187aefad578e4316111e82
+  data.tar.gz: fcc8cd9883129f60bd427c51c3420b7176a3347a0f8f87939669f7cf82d78966
 SHA512:
-  metadata.gz: f0a0e5d9dbeca80d2de42ae1e655191e9aa01c557992b141412cd1dfe86751b3e047b951fc2f6bbf28ea487b7bc8f80d1d173d76c129ca38194cda34f894ef7a
-  data.tar.gz: cf2a9d907e5f35a792c4d9b7f89ea0e0a96b50de499ade69b62afe0108857a7324836021b3ba338d22c05744789b8e252db08a1ae36177f3a5f671003bee6a32
+  metadata.gz: 81611fe51bce4a86cbeccb01362eaf8b1e8e3cc9a618b92206c46cc3afb875779e92962c18582b19df593941f68bac9726f7854902fb811ef97e2edabbd682f4
+  data.tar.gz: 916859633e6ac45c882e5c3ebce09834f4e1f0318fc3eba927549bea54be0f714255fb41f0c5b715f9a4a051a1219aeed7b89ddbc2b14600d0f98c3d79e6947b

data/README.md

@@ -33,7 +33,8 @@ client = Pulse::Downloader::Client.new(
   drop_exitsing_files_in_path: false,
   save_and_dont_return: true,
   report_time: false,
-  progress_bar: true
+  progress_bar: true,
+  scrape_images: true
 )
 
 client.call!

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.25"
+    VERSION = "0.1.26"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -25,7 +25,7 @@ module Pulse
 
       def extract_file_urls(response, custom_path_root, type)
         return [] if response.body.nil? || response.body.empty?
-        (
+        remove_base64(
           extract_all_urls(response, custom_path_root, type) +
             extract_download_links(response, custom_path_root, type) +
             extract_embedded_images(response, custom_path_root, type)
@@ -64,6 +64,12 @@ module Pulse
           .map { |link| add_base_url(link) }
       end
 
+      def remove_base64(urls)
+        urls.reject do |url|
+          url.include?(':image/') || url.include?('base64')
+        end
+      end
+
       def parse_html(raw_html)
         Nokogiri::HTML(raw_html)
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.25
+  version: 0.1.26
 platform: ruby
 authors:
 - trex22
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-05-10 00:00:00.000000000 Z
+date: 2021-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty