pulse-downloader 0.1.14 → 0.1.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eca91e3c21cef3c07b1f1e26c6f7731a605aa54e6861f673bfff6feeabaf0b55
-  data.tar.gz: 57f68ce784c9b7d60342ef45d7830034602eedd35a0ee1afa21ca6cc008f275e
+  metadata.gz: 4ce4c8fd47e92ddc487cd80c36564a21357a3ad79deee82d0af70812c935ab1e
+  data.tar.gz: 3b2dbfc9387fe3c496ae112d3d8c6a7af6cf1b3b0c550fed0da358d9bf798efa
 SHA512:
-  metadata.gz: 390bb819137aee83d9bedcedb06865ca3382f5f36832ed68ebc211d6167fb83944ae0f16fe89f67e876e83a1097eae92916328d9454f0e99f355f79adaabc3c0
-  data.tar.gz: c9e5a5fabb5b33e3251be461b50a8601a0d5cdc6e8d419641d5f441e727f366f841aa3d1d009e0a5817502e388658a1272eea0bf2289db442efa519771b6df03
+  metadata.gz: 1b72add3be0201cfc9c395d790c26c77fc8b8f3383048fc334d8b84695d1d0fd8d9dca17af16242a1ff74c54b8a37e81e40262b90c293507b4668902e23477a5
+  data.tar.gz: b4a2a5087afe6d10ef1c424169862f146f9911d8dd13369ffe86318bab3a083c815a65449375f3a47d1f9864a6de23b5db9417e509285d8647a47637220b9cdb

data/lib/pulse/downloader/client.rb

@@ -8,6 +8,7 @@ module Pulse
 
       attr_reader :url,
         :file_type,
+        :scrape_images,
         :save_data,
         :save_path,
         :read_from_save_path,
@@ -17,7 +18,8 @@ module Pulse
         :report_time,
         :start_time,
         :end_time,
-        :progress_bar
+        :progress_bar,
+        :base_url
 
       # Does not continue downloads-
       # Will only save once the file has been downloaded in memory
@@ -29,6 +31,7 @@ module Pulse
       # TODO: lib/pulse/downloader/file_downloader.rb:13: warning: URI.escape is obsolete
       def initialize(url:,
         file_type:,
+        scrape_images: false,
         save_data: false,
         save_path: '',
         read_from_save_path: false,
@@ -40,6 +43,7 @@ module Pulse
 
         @url = url
         @file_type = file_type
+        @scrape_images = scrape_images
         @save_data = save_data
         @save_path = save_path
         @read_from_save_path = read_from_save_path
@@ -48,6 +52,8 @@ module Pulse
         @save_and_dont_return = save_and_dont_return
         @report_time = report_time
         @progress_bar = progress_bar
+
+        @base_url = get_base_url
       end
 
       def call!
@@ -73,6 +79,16 @@ module Pulse
 
       private
 
+      def get_base_url
+        url_breakdown = url.split('/')
+
+        if url_breakdown.first.include?('https')
+          url_breakdown[2]
+        else
+          url_breakdown.first
+        end
+      end
+
       def get_micro_second_time
         (Time.now.to_f * 1000).to_i
       end

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.14"
+    VERSION = "0.1.19"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -19,18 +19,44 @@ module Pulse
 
       def extract_file_urls(response)
         return [] if response.body.nil? || response.body.empty?
+        (
+          extract_download_links(response) + extract_embedded_images(response)
+        ).uniq
+      end
 
+      def extract_download_links(response)
         parse_html(response.body)
           .css('a')
           .to_a
           .map { |link| link['href'] }
           .compact
           .select { |link| link.include? file_type }
+          .map { |link| add_base_url(link) }
+      end
+
+      def extract_embedded_images(response)
+        return [] unless scrape_images
+
+        parse_html(response.body)
+          .css('img')
+          .to_a
+          .map { |e| e["src"] }
+          .compact
+          .select { |link| link.include? file_type }
+          .map { |link| add_base_url(link) }
       end
 
       def parse_html(raw_html)
         Nokogiri::HTML(raw_html)
       end
+
+      def add_base_url(str)
+        unless str.include?(base_url)
+          "https://#{base_url}#{str}"
+        else
+          str
+        end
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.14
+  version: 0.1.19
 platform: ruby
 authors:
 - trex22
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-05-08 00:00:00.000000000 Z
+date: 2021-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty