pulse-downloader 0.1.24 → 0.1.29

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 13ed97a9caedc58bca6c4247f2d56d7456aecafe0f2eecb65b6c25b5de404fd4
-  data.tar.gz: 0d450684690c49812a63a0aec81f9e9d77e6074898a6ac2be9b7143fe5989adb
+  metadata.gz: ccce085a74c0c4f7710a6e3c7431b191e77bf313a4219ec08e8cd34e78e3340b
+  data.tar.gz: ad2307a1ed1d368fd53a9fa965d7e864a90a263dac689656c49e8a5f62484d9f
 SHA512:
-  metadata.gz: '001790c43ba48c68ad1222d70f1352e5b399384fb7be69c6951ac80fd629ed2009fcc4a1292e23ab712ddaa1a9f206a59b1abd7f965625486160224e4e7a77bb'
-  data.tar.gz: d49f98068eec42477b7069948f4a7c72b37b4e166c6f843b4e7e81763c27eb5e5561115f7b15800e48ed2cd86f5397e6a21f7d3eaf2816a91b017795399fbb9f
+  metadata.gz: 654608db5cd95af6aafde8741864bda84dda783348d0d85baf675418b51c65def47c2ce671910f4313a91c0899784ae92e0d4c6da8e43926fafeaf2053ee2c02
+  data.tar.gz: 520111b61ec241a70321f40707659c09317df3d2b950702be65758bb0a29e6c3bf7880fd7c61e57dafb4450ea2c05b51f834dcb3a2c9ad98ca06f3d61d5c327d

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pulse-downloader (0.1.11)
+    pulse-downloader (0.1.26)
       active_attr (~> 0.15)
       httparty (~> 0.18)
       nokogiri (~> 1.11)
@@ -69,7 +69,7 @@ GEM
       ruby-progressbar
     mocha (1.11.2)
     multi_xml (0.6.0)
-    nokogiri (1.11.3-x86_64-linux)
+    nokogiri (1.11.5-x86_64-linux)
       racc (~> 1.4)
     options (2.3.2)
     progress_bar (1.3.3)

data/README.md

@@ -33,7 +33,8 @@ client = Pulse::Downloader::Client.new(
   drop_exitsing_files_in_path: false,
   save_and_dont_return: true,
   report_time: false,
-  progress_bar: true
+  progress_bar: true,
+  scrape_images: true
 )
 
 client.call!

data/lib/pulse/downloader/client.rb

@@ -13,6 +13,7 @@ module Pulse
         :save_path,
         :read_from_save_path,
         :verify_ssl,
+        :headers,
         :drop_exitsing_files_in_path,
         :save_and_dont_return,
         :report_time,
@@ -36,6 +37,7 @@ module Pulse
         save_path: '',
         read_from_save_path: false,
         verify_ssl: true,
+        headers: nil,
         drop_exitsing_files_in_path: false,
         save_and_dont_return: true,
         report_time: false,
@@ -48,6 +50,7 @@ module Pulse
         @save_path = save_path
         @read_from_save_path = read_from_save_path
         @verify_ssl = verify_ssl
+        @headers = headers
         @drop_exitsing_files_in_path = drop_exitsing_files_in_path
         @save_and_dont_return = save_and_dont_return
         @report_time = report_time

data/lib/pulse/downloader/file_downloader.rb

@@ -8,7 +8,10 @@ module Pulse
 
         @start_time = get_micro_second_time
 
-        file_data = HTTParty.get(escape(compute_file_link(file_path)), verify: verify_ssl)
+        file_data = HTTParty.get(
+          escape(compute_file_link(file_path)),
+          verify: verify_ssl
+        )
 
         @end_time = get_micro_second_time
 

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.24"
+    VERSION = "0.1.29"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -4,7 +4,7 @@ module Pulse
       def fetch_file_paths(custom_path_root=nil)
         @start_time = get_micro_second_time
 
-        response = HTTParty.get(url, verify: verify_ssl)
+        response = HTTParty.get(url, verify: verify_ssl, headers: headers)
 
         @end_time = get_micro_second_time
 
@@ -25,23 +25,35 @@ module Pulse
 
       def extract_file_urls(response, custom_path_root, type)
         return [] if response.body.nil? || response.body.empty?
-        (
-          extract_download_links(response, custom_path_root, type) +
-            extract_embedded_images(response, custom_path_root, type)
+
+        remove_base64(
+          extract_all_urls(response, custom_path_root, type) +
+            extract_download_links(response, type) +
+            extract_embedded_images(response, type)
         ).uniq
       end
 
-      def extract_download_links(response, custom_path_root, type)
+      def extract_all_urls(response, custom_path_root, type)
+        parse_html(response.body)
+          .to_s
+          .split(/\s+/)
+          .find_all { |u| u =~ /^https?:/ }
+          .compact
+          .select { |link| (link.include? type || link.include?(custom_path_root)) }
+          .map { |link| add_base_url(link, custom_path_root) }
+      end
+
+      def extract_download_links(response, type)
         parse_html(response.body)
           .css('a')
           .to_a
           .map { |link| link['href'] }
           .compact
-          .select { |link| (link.include? type || link.include?(custom_path_root)) }
+          .select { |link| (link.include? type) }
           .map { |link| add_base_url(link) }
       end
 
-      def extract_embedded_images(response, custom_path_root, type)
+      def extract_embedded_images(response, type)
         return [] unless scrape_images
 
         parse_html(response.body)
@@ -49,15 +61,34 @@ module Pulse
           .to_a
           .map { |e| e["src"] }
           .compact
-          .select { |link| (link.include? type || link.include?(custom_path_root)) }
+          .select { |link| (link.include? type) }
           .map { |link| add_base_url(link) }
       end
 
+      def remove_artefacts(urls)
+        urls = remove_extra_escape_characters(urls)
+        remove_base64(urls)
+      end
+
+      def remove_extra_escape_characters(urls)
+        urls.map do |url|
+          url.gsub("\">", '')
+        end
+      end
+
+      def remove_base64(urls)
+        urls.reject do |url|
+          url.include?(':image/') || url.include?('base64')
+        end
+      end
+
       def parse_html(raw_html)
         Nokogiri::HTML(raw_html)
       end
 
-      def add_base_url(str)
+      def add_base_url(str, custom_path_root=nil)
+        return str if custom_path_root
+
         if !str.include?('https://') && !str.include?(base_url)
           "https://#{base_url}#{str}"
         else

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.24
+  version: 0.1.29
 platform: ruby
 authors:
 - trex22
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-05-10 00:00:00.000000000 Z
+date: 2021-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty