pulse-downloader 0.1.11 → 0.1.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 608d5fcfe9c84a1c201f54a4a74b64dbe285580e733a77fe89f42c8645b6c2e4
-  data.tar.gz: 3252a32fa9955d1d81b158c3cff4c483329dd1d610a663cd48382a0fe40c1ddc
+  metadata.gz: 2b4edaf42a90782470ab4956cc6d9cf4214d6bb81c03b8e5b55193ac1867330e
+  data.tar.gz: 81da683b9c0aa82f17197a280318b529ab4f6630dd20fa385925739f02165ca2
 SHA512:
-  metadata.gz: f10828967940a47f023dc610e3121f00eebe8962fea1830ccb25c9d84fc8b244749d094cfbfbc8c9ce0495cd572823058fc9557dc665e318a680c6e000703fa7
-  data.tar.gz: 07becda8b5b3c47908c96c6ee5016980ac053c3555b4acac47f1da7d73cf42bcac692351f877cf2ffd2a33d74107a2687614d25a3a8613593d0181bd6159428b
+  metadata.gz: d8771a9ece20d74d44bb8e365d7f51be288ee8a9f5ff5dbec826013aec57998147d58d0e17ffd6ab5c7213f2dc441bfd4b471716f99e8df40e06cf2c7c00ddf0
+  data.tar.gz: 053c925e99adeeb3cae3f277bb0e3ceda00d327ac57fdc76ce25e9a86220d66a291c0f388aa1aebe66a92e7b9b8971409c7b59dc4d188fa87ee4504914b52052

data/lib/pulse/downloader/client.rb

@@ -8,6 +8,7 @@ module Pulse
 
       attr_reader :url,
         :file_type,
+        :scrape_images,
         :save_data,
         :save_path,
         :read_from_save_path,
@@ -29,6 +30,7 @@ module Pulse
       # TODO: lib/pulse/downloader/file_downloader.rb:13: warning: URI.escape is obsolete
       def initialize(url:,
         file_type:,
+        scrape_images: false,
         save_data: false,
         save_path: '',
         read_from_save_path: false,
@@ -40,6 +42,7 @@ module Pulse
 
         @url = url
         @file_type = file_type
+        @scrape_images = scrape_images
         @save_data = save_data
         @save_path = save_path
         @read_from_save_path = read_from_save_path

data/lib/pulse/downloader/file_downloader.rb

@@ -1,8 +1,6 @@
 module Pulse
   module Downloader
     module FileDownloader
-      require 'uri'
-
       # save_path and verify_ssl are defined in client.rb
       def download(file_path, progress_bar=nil)
         raise "save_path is undefined" if save_data && save_path == ''
@@ -10,7 +8,7 @@ module Pulse
 
         @start_time = get_micro_second_time
 
-        file_data = HTTParty.get(URI.escape(compute_file_link(file_path)), verify: verify_ssl)
+        file_data = HTTParty.get(escape(compute_file_link(file_path)), verify: verify_ssl)
 
         @end_time = get_micro_second_time
 
@@ -68,6 +66,37 @@ module Pulse
       def section?(file_path)
         file_path[0] == '#'
       end
+
+      def escape(str)
+        str.gsub!(" ", "%20")
+        str.gsub!("$", "\%24")
+        str.gsub!("&", "\%26")
+        str.gsub!("`", "\%60")
+        # str.gsub!(":", "\%3A")
+        str.gsub!("<", "\%3C")
+        str.gsub!(">", "\%3E")
+        str.gsub!("[", "\%5B")
+        str.gsub!("]", "\%5D")
+        str.gsub!("{", "\%7B")
+        str.gsub!("}", "\%7D")
+        str.gsub!("“", "\%22")
+        str.gsub!('"', "\%22")
+        str.gsub!("+", "\%2B")
+        str.gsub!("#", "\%23")
+        str.gsub!("\%", "\%25")
+        str.gsub!("@", "\%40")
+        # str.gsub!("/", "\%2F")
+        str.gsub!(";", "\%3B")
+        str.gsub!("=", "\%3D")
+        str.gsub!("?", "\%3F")
+        str.gsub!("\\", "\%5C")
+        str.gsub!("^", "\%5E")
+        str.gsub!("|", "\%7C")
+        str.gsub!("~", "\%7E")
+        str.gsub!("‘", "\%27")
+        str.gsub!(",", "\%2C")
+        str
+      end
     end
   end
 end

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.11"
+    VERSION = "0.1.16"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -19,18 +19,44 @@ module Pulse
 
       def extract_file_urls(response)
         return [] if response.body.nil? || response.body.empty?
+        extract_download_links(response) + extract_embedded_images(response)
+      end
 
+      def extract_download_links(response)
         parse_html(response.body)
           .css('a')
           .to_a
           .map { |link| link['href'] }
           .compact
           .select { |link| link.include? file_type }
+          .map { |link| add_base_url(link) }
+      end
+
+      def extract_embedded_images(response)
+        return [] unless scrape_images
+
+        parse_html(response.body)
+          .css('img')
+          .to_a
+          .map { |e| e["src"] }
+          .compact
+          .select { |link| link.include? file_type }
+          .map { |link| add_base_url(link) }
       end
 
       def parse_html(raw_html)
         Nokogiri::HTML(raw_html)
       end
+
+      def add_base_url(str)
+        url_breakdown = url.split('/')
+
+        if url_breakdown.first.include?('https')
+          url_breakdown.third
+        else
+          url_breakdown.first
+        end
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.16
 platform: ruby
 authors:
 - trex22
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-05-08 00:00:00.000000000 Z
+date: 2021-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty