nvx-sds 1.2.1 → 1.3.0

data/README

@@ -25,8 +25,9 @@
 #When you are finished with a session always be sure to log that session out to invalidate the active session token.
 #
 #session.Logout 
+#
 #==== Uploading a file
-#To upload a file you will need to create a session and a SoapUpload object. The call takes a local file and streams it to the remote server. Below is an example of an upload:
+#To upload a file you will need to create a session and an HttpUpload object. The call takes a local file and streams it to the remote server. Below is an example of an upload:
 #
 #session = Session.new("APP-KEY", "USERNAME", "APP NAME", "PASSWORD")
 #
@@ -42,6 +43,19 @@
 #
 #HttpUpload.UploadFile(remote_path, remote_file, local_file, overwrite, session.account_login)
 #
+#==== Downloading a file
+#To download a file you will need to create a session and a HttpDownload object. The call creates a link to a remote Nirvanix file and downloads it to a local file. Below is an example of a download:
+#
+#session = Session.new("APP-KEY", "USERNAME", "APP NAME", "PASSWORD")
+#
+#remote_file = "/TestUpload.txt"
+#
+##Get a file in the current directory
+#
+#local_file = File.expand_path(File.join(File.dirname(__FILE__), '.')) + remote_file
+#
+#HttpDownload.DownloadFile(remote_path, local_file, session.account_login)
+#
 #==== General File operations
 #There are many different file operations that are available once a file has been uploaded through the API. To get to these operations you can retrieve the root folder. The example below shows this and prints out each folder name.
 #

data/RELEASENOTES

@@ -1,6 +1,12 @@
 #= NVX::SDS
 #== A Ruby SDK for Nirvanix
 #=== Release Notes
+#==== May 21, 2009 - 1.3.0
+#
+# * Modified the HttpUpload class to provide more robust retry logic.  Also added callback methods to the HttpUpload.UploadFile method (See RDoc for details).
+# * Created an HttpDownload class to allow downloads with retry logic and callbacks for status and percent complete.  This also downloads the files in chunks and should handle huge files without issues.
+# * Corrected a number of RDoc issues throughout the code.
+#
 #==== May 19, 2009 - 1.2.1
 #
 # * Corrected upload problem with multi-part files in the HttpUpload class.

data/lib/nvx/sds/httpdownload.rb

@@ -0,0 +1,138 @@
+# = Overview
+#
+# The HttpUpload is used to upload files via HTTP POST.  This class can upload a maximum of 256gb.
+module NVX
+    module SDS
+      $:.unshift File.dirname(__FILE__)
+      require 'apiparam'
+      require 'apicommand'
+      require 'apiclasses/responsecodes'
+      require 'net/https'
+      require 'net/http'
+      require 'REXML/Document'
+      require 'uri'
+      
+      # = Overview
+      #
+      # This exception is raised if a transfer fails after retrying 10 times. 
+      class RetryException < RuntimeError 
+        attr :error_message 
+        def initialize(error_message) 
+          @error_message = error_message 
+        end 
+      end
+
+      # = Overview
+      # Retrieves a file from the remote server and saves it to the local file system.
+      class HttpDownload
+        USERAGENT = "Nirvanix Ruby SDK"
+        Newline = "\r\n" unless const_defined? "Newline"
+        BLOCKSIZE = 1024*1024 * 4
+      
+        # = Overview
+        # Retrieves a file from the remote server and saves it to the local file system
+        # The path can be relative or absolute, if you are a master account you can pass the absolute path
+        # to retrieve a child accounts file.
+        #
+        # = Parameters
+        # == path
+        # A path on the Nirvanix file system this can be absolute or regular
+        # == local_path
+        # The local path where the file will be saved
+        # == account_login
+        # the account information from the Session.new that was created with valid login information.
+        # == callback
+        # The callback is an object with two methods for recieving updates while the download is in progress.
+        #
+        #  def percent(percent, bytes_downloaded)
+        #     print "Percent: #{percent} - Bytes: #{bytes_downloaded}\r\n"
+        #  end
+        # 
+        # Returns a the percentage complete along with the number of bytes downloaded so far.
+        #
+        #  def warning(message)
+        #    print "Warning: #{message}\r\n"
+        #  end
+        #
+        # Returns a warning message when there is an internal connectivity issue with the download.
+        #
+        def HttpDownload.DownloadFile(path, local_path, account_login, callback = nil)
+                    # Get the download node this will get the proper node for a path.
+          doc = Transport.execute_command_post(APICommand.GetOptimalURLs, [APIParam.new("filePath", path), APIParam.new("expiration", "#{60*60*24}")], account_login)
+          url = URI.parse(doc.root.elements["//Response/Download/DownloadURL"].get_text.value)
+
+          file = open(local_path, "wb")
+          offset = 0
+          percent_downloaded = 0
+          total_length = 1
+          while offset < total_length
+            begin
+              # download a portion of the file up to blocksize long.
+              response = DownloadChunk(url.host, url.path, offset, BLOCKSIZE, false)
+              # if this is the first request get the total file length based on the Content-Range.
+              if offset == 0
+                range = response['Content-Range']
+                total_length = range[range.index("/") + 1, range.length].to_i
+              end
+              # reset retry_count since this was a successful download.
+              retry_count = 0
+            rescue SocketError, Timeout::Error, Errno::ETIMEDOUT, Errno::EPIPE, IOError, RetryException
+              retry_count += 1
+              if retry_count > 10
+                raise RetryException.new("Connection failure at offset: #{offset}")
+              end
+
+              # If the callback object is available pass a warning specifying the retry count.
+              if !callback.nil? 
+                callback.warning("Connection failure at offset: #{offset}  Retry count: #{retry_count}") 
+              end
+
+              redo
+            end
+            # write the body to the open file.
+            file.write(response.body)
+            # increment the offset by how much was downloaded.
+            offset += response.body.length
+
+            # get percentage complete
+            new_percentage = (offset * 100) / total_length
+            # if the percentage has changed update the callback.
+            if new_percentage > percent_downloaded
+              percent_downloaded = new_percentage
+              if !callback.nil? 
+                callback.percent(percent_downloaded, offset) 
+              end
+            end
+          end
+          file.close
+        end
+
+        # = Overview
+        # Downloads a single chunk from a remote server.
+        def HttpDownload.DownloadChunk(node, path, offset, length, is_secure)
+
+          port = is_secure ? 443 : 80
+              
+          # Create the first http(s) object.
+          if @h.nil?
+            @h = Net::HTTP.new(node, port)
+          end
+          
+          # if you are just testing and do not have a SSL certificate setup properly you can
+          # disable warnings with the below line.  Just uncomment to ignore SSL session warnings.
+          @h.verify_mode = OpenSSL::SSL::VERIFY_NONE if is_secure
+
+          # put the SSL in the correct mode based on the apicommand
+          @h.use_ssl = is_secure
+          headers = {
+                  'Range' => "bytes=#{offset}-#{length + offset}",
+                  'User-Agent' => USERAGENT,
+                  'Content-Length' => "4"
+          }
+
+          # do the command and return the response.
+          return @h.post(path, "\r\n\r\n", headers)
+        end
+    end
+  end
+end

data/lib/nvx/sds/httpupload.rb

@@ -1,34 +1,71 @@
+$:.unshift File.dirname(__FILE__)
+require 'apiparam'
+require 'apicommand'
+require 'apiclasses/responsecodes'
+require 'net/https'
+require 'net/http'
+require 'REXML/Document'
+require 'uri'
+
+
 # = Overview
 #
 # The HttpUpload is used to upload files via HTTP POST.  This class can upload a maximum of 256gb.
 module NVX
     module SDS
 
-        $:.unshift File.dirname(__FILE__)
-        require 'apiparam'
-        require 'apicommand'
-        require 'apiclasses/responsecodes'
-        require 'net/https'
-        require 'net/http'
-        require 'REXML/Document'
-        require 'uri'
-        
         USERAGENT = "Nirvanix Ruby SDK"
         Newline = "\r\n" unless const_defined? "Newline"
         # upload 4 Mb chunks. 4 Mb is usually quite efficient but you may want to change this 
         # to a larger value if you are on a very fast connection or have very high latency.
         BLOCKSIZE = 1024 * 1024 * 4
         
+        # = Overview
+        #
+        # This exception is raised if a transfer fails after retrying 10 times. 
         class RetryException < RuntimeError 
-
+          attr :error_message 
+          def initialize(error_message) 
+            @error_message = error_message 
+          end 
         end
         
         # = Overview
         #
-        # The HttpUpload is used to upload files via HTTP POST.  This class can upload a maximum of 256gb.
+        # The HttpUpload is used to upload files via HTTP POST.  This class can upload a maximum of 256gb.    
         class HttpUpload
-            # Uploads a file to a destination and allows you to call overwrite.
-            def HttpUpload.UploadFile(destination_path, destination_filename, local_path, overwrite, account_login)
+          # = Overview
+          #
+          # The HttpUpload.UploadFile is used to upload files via HTTP POST.  This class can upload a maximum of 256gb.
+          #
+          # = Parameters
+          # == destination_path
+          # The path at Nirvanix where the file will be uploaded
+          # == destination_filename
+          # The remote filename
+          # == local_path
+          # The path to the file that will be uploaded on the local file system.
+          # == overwrite
+          # Determines if the upload should overwrite the file at Nirvanix if it exists.
+          # == account_login
+          # the account information from the Session.new that was created with valid login information.
+          # == callback
+          # The callback is an object with two methods for recieving updates while the download is in progress.
+          #
+          #  def percent(percent, bytes_uploaded)
+          #     print "Percent: #{percent} - Bytes: #{bytes_uploaded}\r\n"
+          #  end
+          # 
+          # Returns a the percentage complete along with the number of bytes uploaded so far.
+          #
+          #  def warning(message)
+          #    print "Warning: #{message}\r\n"
+          #  end
+          #
+          # Returns a warning message when there is an internal connectivity issue with the upload.
+          #        
+          def HttpUpload.UploadFile(destination_path, destination_filename, local_path, overwrite, account_login, 
+              callback = nil)
                 # Get the Upload node passing in the total file size.
                 file_size = File.size(local_path)
                 params = Array.new
@@ -48,30 +85,46 @@ module NVX
                 offset = 0
                 retry_count = 0
                 path = "/upload.ashx?uploadToken=#{upload_token}&destFolderPath=#{destination_path}"
+                percent_uploaded = 0
                 # Loop through the entire file uploading each file part.
                 while !file.eof?
                     # read a chunk of data from the file.
                     file_data = file.read(BLOCKSIZE)
                     # Send a chunk to Nirvanix using the upload token and node.
-                    retry_chunk = false
                     begin
                       tmppath = path
+                      # adjust the path to include rangeOverwrite when trying to write to a previous offset.
                       if retry_count > 0
                         tmppath = path + "&rangeOverwrite=true"
                       end
                       params = post_data(node, tmppath, destination_filename, file_data, offset, file_size, false)
-                    rescue RetryException
+                    # the SystemCallError handles ETIMEOUT, EPIPE, EHOSTDOWN for timeouts and broken pipes.
+                    rescue SocketError, SystemCallError, IOError, RetryException
+                      # set the file position to the previous offset to retry that block.
                       file.pos = offset
                       retry_count += 1
-                      retry_chunk = true
+                      # if the maximum retry count has been reached raise an exception to the outside world.
                       if retry_count == 10
-                        raise RetryException
+                        raise RetryException.new("Connection failure at offset: #{offset}")
+                      end
+                      # If the callback object is available pass a warning specifying the retry count.
+                      if !callback.nil? 
+                        callback.warning("Connection failure at offset: #{offset}  Retry count: #{retry_count}") 
                       end
+                      redo
                     end
+                    # Reset the retry count to 0 since this was transmission was successful.
+                    retry_count = 0
                     # advance offset based on how much data was read.
-                    if !retry_chunk
-                      offset += file_data.length
-                      retry_count = 0
+                    offset += file_data.length
+                    # get percentage complete
+                    new_percentage = (offset * 100) / file_size
+                    # if the percentage has changed update the callback.
+                    if new_percentage > percent_uploaded
+                      percent_uploaded = new_percentage
+                      if !callback.nil? 
+                        callback.percent(percent_uploaded, offset) 
+                      end
                     end
                 end
             end
@@ -102,21 +155,23 @@ module NVX
                 port = is_secure ? 443 : 80
                 
                 # Create the first http(s) object.
-                h = Net::HTTP.new(node, port).start
+                if @h.nil?
+                  @h = Net::HTTP.new(node, port).start
+                end
                 
                 # if you are just testing and do not have a SSL certificate setup properly you can
                 # disable warnings with the below line.  Just uncomment to ignore SSL session warnings.
-                h.verify_mode = OpenSSL::SSL::VERIFY_NONE if is_secure
+                @h.verify_mode = OpenSSL::SSL::VERIFY_NONE if is_secure
                 
                 # put the SSL in the correct mode based on the apicommand
-                h.use_ssl = is_secure
+                @h.use_ssl = is_secure
                 
                 # do the upload and return the response.
                 req = Net::HTTP::Post.new(path, headers)
                 req.content_length = content.length
                 req.content_type = "multipart/form-data; boundary=#{boundary}"
                 req.body = content
-                response = h.request(req)
+                response = @h.request(req)
                 
                 #print "\r\nContent: " + content + "\r\n\r\n"
                 #print "RESPONSE XML: " +  response.body + "\r\n\r\n"

data/lib/nvx/sds/transport.rb

@@ -74,9 +74,7 @@ module NVX
                 return doc
             end
 
-            # Retrieves a file from the remote server and saves it to the local file system
-            # The path can be relative or absolute, if you are a master account you can pass the absolute path
-            # to retrieve a child accounts file.
+            # Obsolete - Use HttpDownload.DownloadFile instead.
             def Transport.DownloadFile(path, local_path, account_login)
                 # Get the download node this will get the proper node for a path.
                 doc = Transport.execute_command_post(APICommand.GetDownloadNodes, [APIParam.new("filePath", path)], account_login)

data/lib/nvx_sds.rb

@@ -18,5 +18,6 @@ require File.join( dir, "masteraccount" )
 # this will be removed soon as the SOAP upload is obsolete.
 require File.join( dir, "apiclasses/soapupload" )
 require File.join( dir, "apiclasses/httpupload" )
+require File.join( dir, "apiclasses/httpdownload" )
 require File.join( dir, "apiclasses/fsfolderlist" )
 require File.join( dir, "apiclasses/metadatainfo" )

data/tests/downloadtest.rb

@@ -0,0 +1,30 @@
+
+require 'test/unit'
+$:.unshift File.dirname(__FILE__) + '/../lib/nvx/sds/'
+require 'session'
+require 'REXML/Document'
+require 'httpdownload'
+require File.dirname(__FILE__) + '/testconfig'
+include REXML
+
+class Downloadtest < Test::Unit::TestCase
+    include NVX::SDS
+
+    def test_HTTPDownloadFile
+        session = Session.new(APP_KEY, USERNAME, APP_NAME, PASSWORD)
+
+        start_time = Time.now
+        local_path = File.expand_path(File.join(File.dirname(__FILE__), '.')) + "/"
+        HttpDownload.DownloadFile(DOWNLOADFILE, local_path + "TEMPFILE.TMP", session.account_login, self)
+        print "\r\nHttpDownloadFile: #{Time.now - start_time}\r\n"
+    end
+
+    def percent(percent, bytes_downloaded)
+      print "Percent: #{percent} - Bytes: #{bytes_downloaded}\r\n"
+    end
+
+    def warning(message)
+      print "Warning: #{message}\r\n"
+    end
+    
+end

data/tests/testconfig.rb

@@ -16,4 +16,7 @@ MASTERPASSWORD = "MASTERPASSWORD"
 # Below is the master account information for testing the master account calls.
 UPLOADPATH = "/"
 UPLOADFILE = "TestUpload.txt"
-UPLOADFILELARGE = "LargeFile.dat"
+UPLOADFILELARGE = "LargeFile.dat"
+
+# this file should exist in your Nirvanix account.
+DOWNLOADFILE = "/TestUpload.txt"

data/tests/uploadtest.rb

@@ -16,7 +16,7 @@ class Uploadtest < Test::Unit::TestCase
         start_time = Time.now
         remote_file = "/TestUpload.txt"
         local_file = File.expand_path(File.join(File.dirname(__FILE__), '.')) + "/" + UPLOADFILE
-        #SoapUpload.UploadFile(UPLOADPATH + UPLOADFILE, local_file, session.account_login)
+        SoapUpload.UploadFile(UPLOADPATH + UPLOADFILE, local_file, session.account_login)
         print "\r\nSoapUploadFile: #{Time.now - start_time}\r\n"
         # remove file after uploading
         #session.DeleteFiles([UPLOADPATH + UPLOADFILE])
@@ -39,18 +39,27 @@ class Uploadtest < Test::Unit::TestCase
 
         start_time = Time.now
         local_file = File.expand_path(File.join(File.dirname(__FILE__), '.')) + "/" + UPLOADFILE
-        HttpUpload.UploadFile(UPLOADPATH, UPLOADFILE, local_file, true, session.account_login)
+        HttpUpload.UploadFile(UPLOADPATH, UPLOADFILE, local_file, true, session.account_login, self)
         print "\r\nHttpUploadFile: #{Time.now - start_time}\r\n"
         # remove file after uploading
-        session.DeleteFiles([UPLOADPATH + UPLOADFILE])
+        #session.DeleteFiles([UPLOADPATH + UPLOADFILE])
+    end
+
+    def percent(percent, bytes_uploaded)
+      print "Percent: #{percent} - Bytes: #{bytes_uploaded}\r\n"
     end
 
+    def warning(message)
+      print "Warning: #{message}\r\n"
+    end
+    
     def test_HTTPUploadFileLarge
         session = Session.new(APP_KEY, USERNAME, APP_NAME, PASSWORD)
 
         start_time = Time.now
         local_file = File.expand_path(File.join(File.dirname(__FILE__), '.')) + "/" + UPLOADFILELARGE
-        HttpUpload.UploadFile(UPLOADPATH, UPLOADFILELARGE, local_file, true, session.account_login)
+        print "\r\nHttpUploadFileLarge Starting...\r\n"
+        HttpUpload.UploadFile(UPLOADPATH, UPLOADFILELARGE, local_file, true, session.account_login, self)
         print "\r\nHttpUploadFileLarge: #{Time.now - start_time}\r\n"
         # remove file after uploading
         session.DeleteFiles([UPLOADPATH + UPLOADFILELARGE])

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: nvx-sds
 version: !ruby/object:Gem::Version 
-  version: 1.2.1
+  version: 1.3.0
 platform: ruby
 authors: 
 - Barry Ruffner
@@ -9,7 +9,7 @@ autorequire: nvx_sds
 bindir: bin
 cert_chain: []
 
-date: 2009-05-19 00:00:00 -07:00
+date: 2009-05-21 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -31,6 +31,7 @@ files:
 - lib/nvx/sds/countries.rb
 - lib/nvx/sds/folder.rb
 - lib/nvx/sds/hosteditem.rb
+- lib/nvx/sds/httpdownload.rb
 - lib/nvx/sds/httpupload.rb
 - lib/nvx/sds/itembase.rb
 - lib/nvx/sds/masteraccount.rb
@@ -56,6 +57,7 @@ files:
 - tests/apicommandtest.rb
 - tests/apiparamtest.rb
 - tests/countriestest.rb
+- tests/downloadtest.rb
 - tests/foldertest.rb
 - tests/fsfolderlisttest.rb
 - tests/masteraccounttest.rb
@@ -103,6 +105,7 @@ test_files:
 - tests/apicommandtest.rb
 - tests/apiparamtest.rb
 - tests/countriestest.rb
+- tests/downloadtest.rb
 - tests/foldertest.rb
 - tests/fsfolderlisttest.rb
 - tests/masteraccounttest.rb