pulse-downloader 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1d8d3d5f2814c1744f58c8a09b118a478b2d1d3f1e92a66df86fd4352284fe4
-  data.tar.gz: 3384a0e37e73e237edf8cd9a2fbdac640473408b7c8e523b8a43ea80529a0680
+  metadata.gz: 143241844753713ded3ebdb7fccf727a20ab2055ad9c40e9936b88f9ecd4a0a5
+  data.tar.gz: 471f4432385de2dc96223ed014e5d3c88adccfda7c98a591b1cd2c915dbe3256
 SHA512:
-  metadata.gz: ee7a13ba93ca48c4306d6cede31c17753610c3496277830a8a5e4bb37b17e141ac231c37ce0bb5166d66db6f7fe6c969e20d8c85d758c3225b5d2b021035ce30
-  data.tar.gz: 6e2999c3df8cbbe876743c10396d59cd8d6c0b1f9723090b910f6594fdd11d0290e30a903b6238e49c36b51fd6f40f04542748c85fcfb34835cdb5f42fb7ab5f
+  metadata.gz: f26e86e5e59e24be532fc542e2de26c33d69dd2e4bb264ef6fe1f28f46d68782d5d0838a80db769912fc202598dff4e40aad10a856e4659e01105c73997b7ed3
+  data.tar.gz: a29a979395ce6b6d7311e7150e22f1ed11a62cd317e19aa42a16241674e7abe33397e8d210882aeb4fa6f7835e3a32e321ccdadf13ecc20219c7a9065a38f9f5

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pulse-downloader (0.1.2)
+    pulse-downloader (0.1.3)
       active_attr (~> 0.15)
       httparty (~> 0.18)
       nokogiri (~> 1.10.9)

data/README.md

@@ -30,6 +30,8 @@ client = Pulse::Downloader::Client.new(
   save_path: '',
   read_from_save_path: false,
   verify_ssl: true,
+  drop_exitsing_files_in_path: false,
+  save_and_dont_return: true
   report_time: false
 )
 ```

data/lib/pulse/downloader.rb

@@ -3,6 +3,7 @@ require 'nokogiri'
 
 require "pulse/downloader/version"
 require 'pulse/downloader/web_page_parser'
+require 'pulse/downloader/file_checker'
 require 'pulse/downloader/file_downloader'
 require 'pulse/downloader/client'
 

data/lib/pulse/downloader/client.rb

@@ -2,6 +2,7 @@ module Pulse
   module Downloader
     class Client
       include ::Pulse::Downloader::WebPageParser
+      include ::Pulse::Downloader::FileChecker
       include ::Pulse::Downloader::FileDownloader
 
       attr_reader :url,
@@ -10,20 +11,35 @@ module Pulse
         :save_path,
         :read_from_save_path,
         :verify_ssl,
+        :drop_exitsing_files_in_path,
         :report_time,
         :start_time,
         :end_time
 
+      # Does not continue downloads-
+      # Will only save once the file has been downloaded in memory
+
+      # TODO: Add in progress bar
       # TODO: Validation
       # TODO: Retry
       # TODO: DNS
-      def initialize(url:, file_type:, save_data: false, save_path: '', read_from_save_path: false, verify_ssl: true, report_time: false)
+      def initialize(url:,
+        file_type:,
+        save_data: false,
+        save_path: '',
+        read_from_save_path: false,
+        verify_ssl: true,
+        drop_exitsing_files_in_path: false,
+        save_and_dont_return: true,
+        report_time: false)
+
         @url = url
         @file_type = file_type
         @save_data = save_data
         @save_path = save_path
         @read_from_save_path = read_from_save_path
         @verify_ssl = verify_ssl
+        @drop_exitsing_files_in_path = drop_exitsing_files_in_path
         @report_time = report_time
       end
 

data/lib/pulse/downloader/file_checker.rb

@@ -0,0 +1,27 @@
+module Pulse
+  module Downloader
+    module FileChecker
+      def file_path_in_file_list?(file_path)
+        return false unless drop_exitsing_files_in_path && save_data
+
+        list_files_in(save_path).include?(compute_save_path(file_path))
+      end
+
+      private
+
+      def compute_save_path(url)
+        "#{save_path}/#{compute_filename(url)}".gsub('//', '/')
+      end
+
+      def compute_filename(file_path)
+        file_path.scan(/[\/]\S+/).last
+      end
+
+      def list_files_in(path)
+        `ls #{path}`.split("\n").map do |filename|
+          "#{path}/#{filename}".gsub('//', '/')
+        end
+      end
+    end
+  end
+end

data/lib/pulse/downloader/file_downloader.rb

@@ -4,6 +4,7 @@ module Pulse
       # save_path and verify_ssl are defined in client.rb
       def download(file_path)
         raise "save_path is undefined" if save_data && save_path == ''
+        return if file_path_in_file_list?(file_path) # skip downloading the file
 
         @start_time = get_micro_second_time
 
@@ -19,6 +20,8 @@ module Pulse
           File.open(compute_save_path(file_path), 'wb') do |file|
             file.write(file_data.body)
           end
+
+          return true if save_and_dont_return
         end
 
         file_data
@@ -36,14 +39,6 @@ module Pulse
 
       private
 
-      def compute_save_path(url)
-        "#{save_path}/#{compute_filename(url)}"
-      end
-
-      def compute_filename(file_path)
-        file_path.scan(/[\/]\S+/).last
-      end
-
       def compute_file_link(file_path)
         if section?(file_path)
           raise 'invalid download path'

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.2"
+    VERSION = "0.1.3"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - trex22
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-15 00:00:00.000000000 Z
+date: 2020-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
@@ -197,6 +197,7 @@ files:
 - bin/setup
 - lib/pulse/downloader.rb
 - lib/pulse/downloader/client.rb
+- lib/pulse/downloader/file_checker.rb
 - lib/pulse/downloader/file_downloader.rb
 - lib/pulse/downloader/version.rb
 - lib/pulse/downloader/web_page_parser.rb