nsrr 0.1.0.beta5 → 0.1.0.rc

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cf83eed2a83df571696e324f97366ea87fc3e490
-  data.tar.gz: 5fcbd1b4d93c3cb2af3e3fbd9648ee4210d994c8
+  metadata.gz: f40189a9d84626e30194f50c78d9b3223c74c817
+  data.tar.gz: 9acdd40aa3ebe199c644607b0932846f0e7577a7
 SHA512:
-  metadata.gz: 7a6db82a874e184a4a3ebedfa19ab0dce5e72aa59145a1b26629103d116b65c456714649cfbd61d85ae301e482895582e0faf8f93b76efd39458c09b094f6674
-  data.tar.gz: eeba970b058397bf73d0bc4557e5143490918fad258f279be6197e5054d08f9f5f9020afd7289fd784e0999c69e004e7a63087f33f8867322b9143e170242435
+  metadata.gz: f869dca518b47dbf5b34c028d4d037661639a166f2670dac658cf08208f960c493870df7f01d25369980460cabb520615824e1da7950c27c17d2adf64c86ee24
+  data.tar.gz: 5a695dd5da13822137dc9f9cc0059813599590a62f29fdc60eeb0e73ff865dfc528117efdfe21faf6c329fe30e0338330415d23692d0a84e69226708b9f3c2a1

data/lib/nsrr/helpers/download_request.rb

@@ -29,22 +29,32 @@ module Nsrr
         end
       end
 
+      # Writes file segments to disk immediately instead of storing in memory
       def get
-        req = Net::HTTP::Get.new(@url.path)
-        response = @http.start do |http|
-          http.request(req)
-        end
-        case response.code when '200'
-          ::File.open(@download_folder, 'wb') do |local_file|
-            local_file.write( response.body )
+        local_file = ::File.open(@download_folder, 'wb')
+        begin
+          partial = true
+          @http.request_get(@url.path) do |response|
+            case response.code when '200'
+              response.read_body do |segment|
+                local_file.write(segment)
+              end
+              @file_size = ::File.size(@download_folder)
+              partial = false
+            when '302'
+              @error = 'Token Not Authorized to Access Specified File'
+            else
+              @error = "#{response.code} #{response.class.name}"
+            end
           end
-          @file_size = ::File.size(@download_folder)
-        when '302'
-          @error = 'Token Not Authorized to Access Specified File'
-        else
-          @error = "#{response.code} #{response.class.name}"
+        rescue => e # Net::ReadTimeout, SocketError
+          @error = "(#{e.class}) #{e.message}"
+        ensure
+          local_file.close()
+          ::File.delete(@download_folder) if partial and ::File.exist?(@download_folder)
         end
       end
+
     end
   end
 end

data/lib/nsrr/models/file.rb

@@ -16,6 +16,8 @@ module Nsrr
         @web_file_size = json['file_size']
         @dataset_slug = json['dataset']
         @file_path = json['file_path']
+        @latest_checksum = ""
+        @latest_file_size = -1
       end
 
       # method:
@@ -30,30 +32,59 @@ module Nsrr
           if md5_matches?(path)
             skip
           else
-            force_download(path, token)
+            force_download(path, token, method)
           end
         else
           if redownload_all
-            force_download(path, token)
+            force_download(path, token, method)
           else
             if file_size_matches?(path)
               skip
             else
-              force_download(path, token)
+              force_download(path, token, method)
             end
           end
         end
       end
 
-      def force_download(path, token)
+      # MD5 or file size checks are now performed after a file is downloaded without error.
+      # If the file check fails, the file is downloaded a second time and rechecked.
+      # If the second download and check fail, the file is marked as failed, and the downloader continues to the subsequent file.
+      def force_download(path, token, method)
         download_folder = ::File.join(Dir.pwd, path.to_s, @name.to_s)
         download_url = "#{Nsrr::WEBSITE}/datasets/#{@dataset_slug}/files/a/#{token}/m/nsrr-gem-v#{Nsrr::VERSION::STRING.gsub('.', '-')}/#{@file_path.to_s}"
         download_request = Nsrr::Helpers::DownloadRequest.new(download_url, download_folder)
         download_request.get
 
+        download_success = false
+
         if download_request.error.to_s == ''
-          puts "    download".colorize(:green) + " #{@name}"
+          # Check to see if the file downloaded correctly
+          # If the file size doesn't match, attempt one additional download
+          download_success = did_download_succeed?(method, path)
+
+          unless download_success
+            download_request = Nsrr::Helpers::DownloadRequest.new(download_url, download_folder)
+            download_request.get
+
+            download_success = did_download_succeed?(method, path)
+          end
+        end
+
+        if download_request.error.to_s == '' and download_success
+          puts "  downloaded".colorize(:green) + " #{@name}"
           download_request.file_size
+        elsif download_request.error.to_s == ''
+          puts "      failed".colorize(:red) + " #{@name}"
+          if method == 'fast'
+            puts "             File size mismatch, expected: #{@web_file_size}"
+            puts "                                   actual: #{@latest_file_size}"
+          else
+            puts "             File checksum mismatch, expected: #{@web_checksum}"
+            puts "                                       actual: #{@latest_checksum}"
+          end
+          ::File.delete(download_folder) if ::File.exist?(download_folder)
+          'fail'
         else
           puts "      failed".colorize(:red) + " #{@name}"
           puts "             #{download_request.error}"
@@ -72,11 +103,12 @@ module Nsrr
 
       def local_checksum(path)
         download_folder = ::File.join(Dir.pwd, path.to_s, self.name.to_s)
-        if ::File.exist?(download_folder)
+        @latest_checksum = if ::File.exist?(download_folder)
           Digest::MD5.file(download_folder).hexdigest
         else
           ""
         end
+        @latest_checksum
       end
 
       def file_size_matches?(path)
@@ -85,11 +117,20 @@ module Nsrr
 
       def local_filesize(path)
         download_folder = ::File.join(Dir.pwd, path.to_s, self.name.to_s)
-        if ::File.exist?(download_folder)
+        @latest_file_size = if ::File.exist?(download_folder)
           ::File.size(download_folder)
         else
           -1
         end
+        @latest_file_size
+      end
+
+      def did_download_succeed?(method, path)
+        if method == 'fast'
+          file_size_matches?(path)
+        else
+          md5_matches?(path)
+        end
       end
 
     end

data/lib/nsrr/version.rb

@@ -3,7 +3,7 @@ module Nsrr
     MAJOR = 0
     MINOR = 1
     TINY = 0
-    BUILD = "beta5" # nil, "pre", "rc", "rc2"
+    BUILD = "rc" # nil, "pre", "rc", "rc2"
 
     STRING = [MAJOR, MINOR, TINY, BUILD].compact.join('.')
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nsrr
 version: !ruby/object:Gem::Version
-  version: 0.1.0.beta5
+  version: 0.1.0.rc
 platform: ruby
 authors:
 - Remo Mueller
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-29 00:00:00.000000000 Z
+date: 2014-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake