pulse-downloader 0.1.10 → 0.1.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 73e18a59a2534ac4b716222495792c2624a827aa220647579321552baa31e451
-  data.tar.gz: 25c64012141ae32139ff3fb6eed9ed8e388a5ce001580127dc28040135731cae
+  metadata.gz: 4d2a34a08bede3b49a3d3629fc2d785d6337dc62702dbeb86742323b4baa11aa
+  data.tar.gz: 29e276ed758b83ff22453f7774afd4e28aec0b6eac3436302287e239e73e0d73
 SHA512:
-  metadata.gz: 1423d3cf884fca38e31d5b54133e5449f365f848f6a1d441edfa053031f8cf5a6b19f99b9b0bbdc2d8d71126fa5294051d101f38ca7d43a7440105644d2ed1f7
-  data.tar.gz: b86aac8791cb20f12f1e9b992e650614e4f895658c492b61288f581c1aa43148fd21a76612a73df3edd660b5e6d0c3589659c8c97243abc126ab973a1a5bcf07
+  metadata.gz: ddd1e433cc5be43243d83867107089945aa756328fa3e3fb4d239b682bfa88c29563e8401a3dccadd266d87169129ae994132a178179740c91520b790a7209c8
+  data.tar.gz: cb3c310e893e3bb8c095d1e42b299f5bb3c9f95128b1657f65e066e7c60d154b22947f5d70d8e675f71438e7b0fe5c83cecef5a8806ba8b1a08e29f4a8f4bfb9

data/Gemfile.lock

@@ -1,23 +1,24 @@
 PATH
   remote: .
   specs:
-    pulse-downloader (0.1.4)
+    pulse-downloader (0.1.11)
       active_attr (~> 0.15)
       httparty (~> 0.18)
       nokogiri (~> 1.11)
+      progress_bar (~> 1.3.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (6.1.3.1)
-      actionview (= 6.1.3.1)
-      activesupport (= 6.1.3.1)
+    actionpack (6.1.3.2)
+      actionview (= 6.1.3.2)
+      activesupport (= 6.1.3.2)
       rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actionview (6.1.3.1)
-      activesupport (= 6.1.3.1)
+    actionview (6.1.3.2)
+      activesupport (= 6.1.3.2)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
@@ -26,9 +27,9 @@ GEM
       actionpack (>= 3.0.2, < 7.0)
       activemodel (>= 3.0.2, < 7.0)
       activesupport (>= 3.0.2, < 7.0)
-    activemodel (6.1.3.1)
-      activesupport (= 6.1.3.1)
-    activesupport (6.1.3.1)
+    activemodel (6.1.3.2)
+      activesupport (= 6.1.3.2)
+    activesupport (6.1.3.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -45,6 +46,7 @@ GEM
     crass (1.0.6)
     erubi (1.10.0)
     hashdiff (1.0.1)
+    highline (2.0.3)
     httparty (0.18.1)
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
@@ -69,6 +71,10 @@ GEM
     multi_xml (0.6.0)
     nokogiri (1.11.3-x86_64-linux)
       racc (~> 1.4)
+    options (2.3.2)
+    progress_bar (1.3.3)
+      highline (>= 1.6, < 3)
+      options (~> 2.3.0)
     pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)

data/lib/pulse/downloader/client.rb

@@ -8,6 +8,7 @@ module Pulse
 
       attr_reader :url,
         :file_type,
+        :scrape_images,
         :save_data,
         :save_path,
         :read_from_save_path,
@@ -29,6 +30,7 @@ module Pulse
       # TODO: lib/pulse/downloader/file_downloader.rb:13: warning: URI.escape is obsolete
       def initialize(url:,
         file_type:,
+        scrape_images: false,
         save_data: false,
         save_path: '',
         read_from_save_path: false,
@@ -40,6 +42,7 @@ module Pulse
 
         @url = url
         @file_type = file_type
+        @scrape_images = scrape_images
         @save_data = save_data
         @save_path = save_path
         @read_from_save_path = read_from_save_path

data/lib/pulse/downloader/file_downloader.rb

@@ -1,8 +1,6 @@
 module Pulse
   module Downloader
     module FileDownloader
-      require 'uri'
-
       # save_path and verify_ssl are defined in client.rb
       def download(file_path, progress_bar=nil)
         raise "save_path is undefined" if save_data && save_path == ''
@@ -10,7 +8,7 @@ module Pulse
 
         @start_time = get_micro_second_time
 
-        file_data = HTTParty.get(URI.escape(compute_file_link(file_path)), verify: verify_ssl)
+        file_data = HTTParty.get(escape(compute_file_link(file_path)), verify: verify_ssl)
 
         @end_time = get_micro_second_time
 
@@ -68,6 +66,37 @@ module Pulse
       def section?(file_path)
         file_path[0] == '#'
       end
+
+      def escape(str)
+        str.gsub!(" ", "%20")
+        str.gsub!("$", "\%24")
+        str.gsub!("&", "\%26")
+        str.gsub!("`", "\%60")
+        # str.gsub!(":", "\%3A")
+        str.gsub!("<", "\%3C")
+        str.gsub!(">", "\%3E")
+        str.gsub!("[", "\%5B")
+        str.gsub!("]", "\%5D")
+        str.gsub!("{", "\%7B")
+        str.gsub!("}", "\%7D")
+        str.gsub!("“", "\%22")
+        str.gsub!('"', "\%22")
+        str.gsub!("+", "\%2B")
+        str.gsub!("#", "\%23")
+        str.gsub!("\%", "\%25")
+        str.gsub!("@", "\%40")
+        # str.gsub!("/", "\%2F")
+        str.gsub!(";", "\%3B")
+        str.gsub!("=", "\%3D")
+        str.gsub!("?", "\%3F")
+        str.gsub!("\\", "\%5C")
+        str.gsub!("^", "\%5E")
+        str.gsub!("|", "\%7C")
+        str.gsub!("~", "\%7E")
+        str.gsub!("‘", "\%27")
+        str.gsub!(",", "\%2C")
+        str
+      end
     end
   end
 end

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.10"
+    VERSION = "0.1.15"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -19,7 +19,10 @@ module Pulse
 
       def extract_file_urls(response)
         return [] if response.body.nil? || response.body.empty?
+        extract_download_links(response) + extract_embedded_images(response)
+      end
 
+      def extract_download_links(response)
         parse_html(response.body)
           .css('a')
           .to_a
@@ -28,6 +31,16 @@ module Pulse
           .select { |link| link.include? file_type }
       end
 
+      def extract_embedded_images(response)
+        parse_html(response.body)
+          .css('img')
+          .to_a
+          .map { |e| e["src"] }
+          .compact
+          .select { |link| link.include? file_type }
+          .select { |link| link.include? "https://" }
+      end
+
       def parse_html(raw_html)
         Nokogiri::HTML(raw_html)
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.1.15
 platform: ruby
 authors:
 - trex22
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-04-22 00:00:00.000000000 Z
+date: 2021-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty