pulse-downloader 0.1.21 → 0.1.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db7034531cb0590f5975013cb817d4093d603d93a7d89d16d099203c063125f9
-  data.tar.gz: e6277a95389810ebcd11957645318daf6829d05d44bea2e73bd2b05fcdda56d1
+  metadata.gz: b59437d8b4d75cbb7c68010644dbb046f6860e9c37187aefad578e4316111e82
+  data.tar.gz: fcc8cd9883129f60bd427c51c3420b7176a3347a0f8f87939669f7cf82d78966
 SHA512:
-  metadata.gz: 72109c3a8509f1da9f1d2c25c1958d8e5cc897e03224dda26ce6ae5da50ac96c754ec502fa44a2bbb8a1ae99fe34b30841f6eb2c3dfa64181f32b3b6c9866e9c
-  data.tar.gz: 780396541173f9fd8e1ce8044522bc876a3b55123adf169a8a62161252bf3e5905f09b38741c4faa70a064a8a89f01192005ff5a4ad924725b83a5b31c9ae9e1
+  metadata.gz: 81611fe51bce4a86cbeccb01362eaf8b1e8e3cc9a618b92206c46cc3afb875779e92962c18582b19df593941f68bac9726f7854902fb811ef97e2edabbd682f4
+  data.tar.gz: 916859633e6ac45c882e5c3ebce09834f4e1f0318fc3eba927549bea54be0f714255fb41f0c5b715f9a4a051a1219aeed7b89ddbc2b14600d0f98c3d79e6947b

data/README.md

@@ -33,7 +33,8 @@ client = Pulse::Downloader::Client.new(
   drop_exitsing_files_in_path: false,
   save_and_dont_return: true,
   report_time: false,
-  progress_bar: true
+  progress_bar: true,
+  scrape_images: true
 )
 
 client.call!

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.21"
+    VERSION = "0.1.26"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -1,7 +1,7 @@
 module Pulse
   module Downloader
     module WebPageParser
-      def fetch_file_paths
+      def fetch_file_paths(custom_path_root=nil)
         @start_time = get_micro_second_time
 
         response = HTTParty.get(url, verify: verify_ssl)
@@ -12,29 +12,47 @@ module Pulse
           print_time
         end
 
-        extract_file_urls(response)
+        if file_type.is_a?(Array)
+          file_type.flat_map do |type|
+            extract_file_urls(response, custom_path_root, type)
+          end
+        else
+          extract_file_urls(response, custom_path_root, file_type)
+        end
       end
 
       private
 
-      def extract_file_urls(response)
+      def extract_file_urls(response, custom_path_root, type)
         return [] if response.body.nil? || response.body.empty?
-        (
-          extract_download_links(response) + extract_embedded_images(response)
+        remove_base64(
+          extract_all_urls(response, custom_path_root, type) +
+            extract_download_links(response, custom_path_root, type) +
+            extract_embedded_images(response, custom_path_root, type)
         ).uniq
       end
 
-      def extract_download_links(response)
+      def extract_all_urls(response, custom_path_root, type)
+        parse_html(response.body)
+          .to_s
+          .split(/\s+/)
+          .find_all { |u| u =~ /^https?:/ }
+          .compact
+          .select { |link| (link.include? type || link.include?(custom_path_root)) }
+          .map { |link| add_base_url(link) }
+      end
+
+      def extract_download_links(response, custom_path_root, type)
         parse_html(response.body)
           .css('a')
           .to_a
           .map { |link| link['href'] }
           .compact
-          .select { |link| link.include? file_type }
+          .select { |link| (link.include? type || link.include?(custom_path_root)) }
           .map { |link| add_base_url(link) }
       end
 
-      def extract_embedded_images(response)
+      def extract_embedded_images(response, custom_path_root, type)
         return [] unless scrape_images
 
         parse_html(response.body)
@@ -42,16 +60,22 @@ module Pulse
           .to_a
           .map { |e| e["src"] }
           .compact
-          .select { |link| link.include? file_type }
+          .select { |link| (link.include? type || link.include?(custom_path_root)) }
           .map { |link| add_base_url(link) }
       end
 
+      def remove_base64(urls)
+        urls.reject do |url|
+          url.include?(':image/') || url.include?('base64')
+        end
+      end
+
       def parse_html(raw_html)
         Nokogiri::HTML(raw_html)
       end
 
       def add_base_url(str)
-        if !str.include?('https://') || !str.include?(base_url)
+        if !str.include?('https://') && !str.include?(base_url)
           "https://#{base_url}#{str}"
         else
           str

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.21
+  version: 0.1.26
 platform: ruby
 authors:
 - trex22
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-05-10 00:00:00.000000000 Z
+date: 2021-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty