cnvrg 1.11.1 → 1.11.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa004590a2756120371db2d6322a564e364076453f9d7b858e83c94e2068b4b9
-  data.tar.gz: d23ee94fbbd6bfb6b09b303d71f4fa5c646d6fc76cb6d049e96f7d3dddfff348
+  metadata.gz: 1402d0759d20d1c1d9182cebc35fe5539c14a8a1bb2c4eeddf791f43ff768082
+  data.tar.gz: 4146973b1a98ea3d26af0efb96ee481093e29fe9fce6fb88c54a539b34e5c0cf
 SHA512:
-  metadata.gz: 0f35da8bf64a49b8ddfa3baea029ad08aa29665632763799b550886cc2647aa81933f967ec6bb5b2b41ab536d1784b265b9d139c0151ee92cbee02dcc33e47a0
-  data.tar.gz: e355bf1a4081462895a278998010b8ce587d9d4e065a63df3d8cab65948abc09468fb84511e665f29354782dcf94bc9298d701a03592d5daf5503dbb6228afc9
+  metadata.gz: 53951eb4b7f82322640e75fdeae5ece07bb7189cc5b81257a9a78fd1398ab04c43ef0ed2784c1963f8d53dd983f874f2ed1ccc0ac4953ffced37deb75953d679
+  data.tar.gz: 8a046e229d0d14a2d36cb896c467aff3599e98b0e1cfb93e96fb72e80342751a47df4e1a90a57ff62e09ed92f1720afb783fe354dd34a5e06767e98c5bd8bdb6

data/lib/cnvrg/cli.rb

@@ -896,10 +896,12 @@ module Cnvrg
             return
           end
 
+
           if only_tree
-            success = Dataset.clone_tree(commit: commit, dataset_home: dataset_home)
-            return if success
+            Dataset.clone_tree(commit: commit, dataset_home: dataset_home)
+            return
           end
+
           commit = response["result"]["commit"]
           files_count = response["result"]["file_count"]
           files = @files.get_clone_chunk(commit: commit)
@@ -911,6 +913,7 @@ module Cnvrg
                                            :total => files_count,
                                            :autofinish => true)
 
+          Dataset.clone_tree(commit: commit, dataset_home: dataset_home, progressbar: progressbar)
 
           while files['keys'].length > 0
             Cnvrg::Logger.log_info("download multiple files, #{downloaded_files.size} files downloaded")
@@ -969,6 +972,7 @@ module Cnvrg
 
         if Dataset.blank_clone(owner, dataset_name, dataset_slug)
           dataset = Dataset.new(dataset_home)
+          downloader = dataset.get_storage_client
           log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
           parallel_options = {
               :progress => {
@@ -991,15 +995,20 @@ module Cnvrg
               relative_path_dir = relative_path_dir.join("/")
               abs_path = dataset_home + "/" + relative_path_dir
               abs_path = dataset_home if flatten
+              fullpath = abs_path + "/" + file_name
 
               begin
-                FileUtils.mkdir_p(abs_path) unless File.exist? (abs_path + "/" + file_name)
+                FileUtils.mkdir_p(abs_path) unless File.exist? (fullpath)
               rescue
                 log_message("Could not create directory: #{abs_path}", Thor::Shell::Color::RED)
                 exit(1)
               end
               begin
-                File.write "#{abs_path}/#{file_name}", open(f["url"]).read unless File.exist? (abs_path + "/" + file_name)
+                unless File.exist?(fullpath)
+                  downloader.safe_operation("#{abs_path}/#{file_name}") do
+                    File.open(fullpath, "w") { |f| f.write open(f["url"]).read }
+                  end
+                end
               rescue => e
                 log_message("Could not download file: #{f["fullpath"]}", Thor::Shell::Color::RED)
                 exit(1)

data/lib/cnvrg/datafiles.rb

@@ -1338,7 +1338,7 @@ module Cnvrg
             #   Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
             #   next
             # end
-            resp = @downloader.download(storage_path, local_path)
+            resp = @downloader.safe_download(storage_path, local_path)
             Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
           rescue => e
             Cnvrg::Logger.log_error(e)

data/lib/cnvrg/dataset.rb

@@ -471,23 +471,38 @@ module Cnvrg
       response.to_json
     end
 
-    def self.clone_tree(commit: 'latest', dataset_home: nil)
+    def self.clone_tree(commit: 'latest', dataset_home: nil, progressbar: nil)
       @dataset = Cnvrg::Dataset.new(dataset_home)
       @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
       trees = @files.get_trees(commit: commit)
       return false if trees.nil?
-      pb = ProgressBar.create(:title => "Download Progress",
-                              :progress_mark => '=',
-                              :format => "%b>>%i| %p%% %t",
-                              :starting_at => 0,
-                              :total => trees.size,
-                              :autofinish => true)
+
+      if progressbar
+        pb = progressbar
+        pb.total += trees.size
+      else
+        pb = ProgressBar.create(
+          :title => "Download Progress",
+          :progress_mark => '=',
+          :format => "%b>>%i| %p%% %t",
+          :starting_at => 0,
+          :total => trees.size,
+          :autofinish => true
+        )
+      end
+
       trees.each do |tree|
         pb.progress += 1
         @files.download_dir(dataset_home, tree)
       end
-      pb.finish
-      @dataset.write_success
+
+      unless progressbar
+        # if progessbar sent it means that its not only tree so we dont want to finish the progress bar
+        # and we dont want to write success
+
+        pb.finish
+        @dataset.write_success
+      end
       true
     end
 

data/lib/cnvrg/downloader/client.rb

@@ -30,10 +30,14 @@ module Cnvrg
         file.gsub(prefix, '').gsub(/^\/*/, '')
       end
 
-      def download(storage_path, local_path)
+      def download(storage_path, local_path, decrypt: true)
         ### need to be implemented..
       end
 
+      def safe_download(storage_path, local_path, decrypt: true)
+        safe_operation(local_path) { self.download(storage_path, local_path, decrypt: decrypt) }
+      end
+
       def upload(storage_path, local_path)
         ### need to be implemented..
       end
@@ -51,17 +55,34 @@ module Cnvrg
       end
 
       def safe_upload(storage_path, local_path)
+        safe_operation(local_path) { self.upload(storage_path, local_path) }
+      end
+
+      def self.factory(params)
+        params = params.as_json
+        case params["storage"]
+        when 's3', 'minio'
+          return Cnvrg::Downloader::Clients::S3Client.new(sts_path: params["path_sts"], access_key: params["sts_a"], secret: params["sts_s"], session_token: params["sts_st"], region: params["region"], bucket: params["bucket"], encryption: params["encryption"], endpoint: params["endpoint"], storage: params["storage"])
+        when 'azure'
+          azure_params = params.symbolize_keys.slice(*[:storage_account_name, :storage_access_key, :container, :sts])
+          return Cnvrg::Downloader::Clients::AzureClient.new(**azure_params)
+        when 'gcp'
+          return Cnvrg::Downloader::Clients::GcpClient.new(project_id: params["project_id"], credentials: params["credentials"], bucket_name: params["bucket_name"], sts: params["sts"])
+        end
+      end
+
+      def safe_operation(local_path)
         n = 1
         error = nil
         while n <= RETRIES
           begin
-            self.upload(storage_path, local_path)
+            yield
             error = nil
             break
           rescue => e
             backoff_time_seconds = backoff_time(n)
 
-            message = "Got error: #{e.class.name} with message: #{e.message} while uploading a single file: #{local_path}, retry: #{n} of: #{RETRIES}"
+            message = "Got error: #{e.class.name} with message: #{e.message} while uploading / downloading a single file: #{local_path}, retry: #{n} of: #{RETRIES}"
             if n < RETRIES
               message += ", next retry in: #{backoff_time_seconds} seconds"
             else
@@ -79,19 +100,6 @@ module Cnvrg
         true
       end
 
-      def self.factory(params)
-        params = params.as_json
-        case params["storage"]
-        when 's3', 'minio'
-          return Cnvrg::Downloader::Clients::S3Client.new(sts_path: params["path_sts"], access_key: params["sts_a"], secret: params["sts_s"], session_token: params["sts_st"], region: params["region"], bucket: params["bucket"], encryption: params["encryption"], endpoint: params["endpoint"], storage: params["storage"])
-        when 'azure'
-          azure_params = params.symbolize_keys.slice(*[:storage_account_name, :storage_access_key, :container, :sts])
-          return Cnvrg::Downloader::Clients::AzureClient.new(**azure_params)
-        when 'gcp'
-          return Cnvrg::Downloader::Clients::GcpClient.new(project_id: params["project_id"], credentials: params["credentials"], bucket_name: params["bucket_name"], sts: params["sts"])
-        end
-      end
-
       private
 
       def random_number_milliseconds

data/lib/cnvrg/downloader/clients/gcp_client.rb

@@ -31,7 +31,7 @@ module Cnvrg
           @tempfile = t
         end
 
-        def download(storage_path, local_path)
+        def download(storage_path, local_path, decrypt: true)
           prepare_download(local_path)
           file = @bucket.file(decrypt(storage_path))
           file.download local_path

data/lib/cnvrg/files.rb

@@ -115,7 +115,7 @@ module Cnvrg
         #upload files
         blob_id_chunk = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
           begin
-            Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(parsed_chunk_of_files[file]))}
+            upload_single_file(files[file].merge(parsed_chunk_of_files[file]))
           rescue => e
             Cnvrg::CLI.log_message("Failed to upload #{file}: #{e.message}", 'red') unless suppress_exceptions
             Cnvrg::Logger.log_error(e)
@@ -188,7 +188,7 @@ module Cnvrg
     def upload_single_file(file)
       path = file['path']
       absolute_path = file[:absolute_path]
-      @client.upload(path, absolute_path)
+      @client.safe_upload(path, absolute_path)
     end
 
     def parse_file(file)
@@ -789,7 +789,7 @@ module Cnvrg
               end
               local_path = project_home+"/"+file_path
               storage_path = f["path"]
-              @client.download(storage_path, local_path)
+              @client.safe_download(storage_path, local_path)
               progress.progress += 1 if progress.present?
               download_succ_count += 1
             rescue => e
@@ -962,7 +962,7 @@ module Cnvrg
 
     def download_file(file_path: '', key: '', iv: '', bucket: '', path: '', client: nil)
       local_path = @project_home+"/"+file_path
-      @client.download(path, local_path)
+      @client.safe_download(path, local_path)
     end
 
     def delete(file)

data/lib/cnvrg/storage.rb

@@ -34,7 +34,7 @@ module Cnvrg
         @element.get_clone_chunk(commit: commit, chunk_size: params[:limit], offset: params[:offset])
       end
       action = Proc.new do |storage, local|
-        @client.download(storage, local)
+        @client.safe_download(storage, local)
       end
 
       @stats = @element.get_stats

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.11.1'
+  VERSION = '1.11.7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.11.1
+  version: 1.11.7
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-25 00:00:00.000000000 Z
+date: 2020-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -453,7 +453,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.4
 signing_key:
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.