pulse-downloader 0.1.15 → 0.1.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4d2a34a08bede3b49a3d3629fc2d785d6337dc62702dbeb86742323b4baa11aa
-  data.tar.gz: 29e276ed758b83ff22453f7774afd4e28aec0b6eac3436302287e239e73e0d73
+  metadata.gz: 8a2d2732c3a0307e7dffaf1ec3b60691e7b54ca3de4c4084325908f6fdf1b801
+  data.tar.gz: 06b3a802a55f186f8fccceec0e29d6b6e3dbade7be776699a6a327ccacf68060
 SHA512:
-  metadata.gz: ddd1e433cc5be43243d83867107089945aa756328fa3e3fb4d239b682bfa88c29563e8401a3dccadd266d87169129ae994132a178179740c91520b790a7209c8
-  data.tar.gz: cb3c310e893e3bb8c095d1e42b299f5bb3c9f95128b1657f65e066e7c60d154b22947f5d70d8e675f71438e7b0fe5c83cecef5a8806ba8b1a08e29f4a8f4bfb9
+  metadata.gz: 27c5f745685fbb5b2c207113b4fe28eb97b8638551afb931e3be1e4d9f073c5959ba0cc1ed59db44fbdfaae26218840ca8663da95fba37ef7c0cf18a37f2aefc
+  data.tar.gz: 3a06bbe0fafe7329eec9ba1d90e483da181f309915b06bfb67d49c3de9f1c96402047d775f672ed172979ca210d988e7063acd5c53860dc61bfcf508a6f64457

data/lib/pulse/downloader/client.rb

@@ -18,7 +18,8 @@ module Pulse
         :report_time,
         :start_time,
         :end_time,
-        :progress_bar
+        :progress_bar,
+        :base_url
 
       # Does not continue downloads-
       # Will only save once the file has been downloaded in memory
@@ -51,6 +52,8 @@ module Pulse
         @save_and_dont_return = save_and_dont_return
         @report_time = report_time
         @progress_bar = progress_bar
+
+        @base_url = get_base_url
       end
 
       def call!
@@ -76,6 +79,16 @@ module Pulse
 
       private
 
+      def get_base_url
+        url_breakdown = url.split('/')
+
+        if url_breakdown.first.include?('https')
+          url_breakdown[2]
+        else
+          url_breakdown.first
+        end
+      end
+
       def get_micro_second_time
         (Time.now.to_f * 1000).to_i
       end

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.15"
+    VERSION = "0.1.20"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -19,7 +19,9 @@ module Pulse
 
       def extract_file_urls(response)
         return [] if response.body.nil? || response.body.empty?
-        extract_download_links(response) + extract_embedded_images(response)
+        (
+          extract_download_links(response) + extract_embedded_images(response)
+        ).uniq
       end
 
       def extract_download_links(response)
@@ -29,21 +31,34 @@ module Pulse
           .map { |link| link['href'] }
           .compact
           .select { |link| link.include? file_type }
+          .map { |link| add_base_url(link) }
       end
 
       def extract_embedded_images(response)
+        return [] unless scrape_images
+
         parse_html(response.body)
           .css('img')
           .to_a
           .map { |e| e["src"] }
           .compact
           .select { |link| link.include? file_type }
-          .select { |link| link.include? "https://" }
+          .map { |link| add_base_url(link) }
       end
 
       def parse_html(raw_html)
         Nokogiri::HTML(raw_html)
       end
+
+      def add_base_url(str)
+        unless str.include?('https://')
+          str
+        unless str.include?(base_url)
+          "https://#{base_url}#{str}"
+        else
+          str
+        end
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.15
+  version: 0.1.20
 platform: ruby
 authors:
 - trex22