cnvrg 1.6.36 → 1.6.38

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c5e60c1cc7c0af133390dd106ee666db3787b6c995b54d5ef97edff3d073ff14
-  data.tar.gz: 3d37f71642874898b131812fdeb1ca7be3b7eebddc3a8aa521245c31eaa47a0b
+  metadata.gz: e5953510af9633c925be99ac1463aef31667eed5f83f5b0c610672ab9fe375cb
+  data.tar.gz: 7bd8fce2d93a20ed695e4c202f2b29092de84bf90332385a3921844046a0c591
 SHA512:
-  metadata.gz: dc82247f7d78186aa0b35084d83b0d4ae72608f0c765727ee57bebcec76d6bc9c84fad2795c7c7ed7ba6ac64921df49c9c45cd3536128b11b8a5b8203e8e0040
-  data.tar.gz: 896d84a1615dd3a698c5e78ce92d7422fca9e6f21e1c7e9ca6011ab315437b39b2f3f5fb0790d683397e2ba6bdaaad4185156255e615e298471b50949201f9d2
+  metadata.gz: 8b93cd899060800095895967dbe2b8f3391e2d77b6f29ea05cd6651de4d26de2ba24fdaedaa8dafad82a22e250d95abf7651d2b77afe2dcf68b883461774c965
+  data.tar.gz: a4dc324e6f58b628013bb00278a74233f6b2c6e4ffa260e9edf74ea2557ca144907449078defb1abdb22ffe411fdde196b2e029bd66e67233140cf54d6caf67d

data/lib/cnvrg/cli.rb

@@ -2211,6 +2211,7 @@ module Cnvrg
     method_option :return_id, :type => :boolean, :aliases => ["-r", "--return_id"], :default => false
     method_option :files, :type => :string, :aliases => ["--files"], :default => nil
     method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
+    method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
     method_option :job_slug, :type => :string, :aliases => ["--job"], :default => nil, :hide=>true
     method_option :job_type, :type => :string, :aliases => [ "--job_type"], :default => nil, :hide=>true
 
@@ -2235,19 +2236,21 @@ module Cnvrg
           spec_files_to_upload = spec_files_to_upload.split(",")
         end
         if @project.is_git
+          list = []
           git_output_dir = options["output_dir"] || output_dir
           if git_output_dir.present?
             if git_output_dir.ends_with? "/"
               git_output_dir = git_output_dir[0..-2]
             end
             list = @project.generate_output_dir(git_output_dir)
-            spec_files_to_upload = list
-            if spec_files_to_upload.blank?
-              log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
-              return true
-            end
-            force = true
           end
+          list += @project.generate_git_diff if options["git_diff"]
+          spec_files_to_upload = list
+          if spec_files_to_upload.blank?
+            log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
+            return true
+          end
+          force = true
         end
 
         if ignore.nil? or ignore.empty?
@@ -2892,6 +2895,7 @@ module Cnvrg
     method_option :job_type, :type => :string, :aliases => ["-jt", "--job_type"], :default => nil
     method_option :files, :type => :string, :aliases => ["--files"], :default => nil
     method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
+    method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
     def sync(direct = true)
       verify_logged_in(true) if direct
       @project = Project.new(get_project_home)
@@ -2912,7 +2916,7 @@ module Cnvrg
       end
       invoke :upload, [false, true,  direct, "",in_exp,options[:force],  options["output_dir"],job_type, job_slug ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
              :ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"],
-             :files => options["files"], :output_dir => options["output_dir"], :job_slug => job_slug, :job_type => job_type
+             :files => options["files"], :output_dir => options["output_dir"], :job_slug => job_slug, :job_type => job_type, :git_diff=> options["git_diff"]
     end
 
     desc 'run cmd', 'Runs an experiment'

data/lib/cnvrg/datafiles.rb

@@ -9,8 +9,8 @@ module Cnvrg
   class Datafiles
     ParallelThreads ||= Cnvrg::Helpers.parallel_threads
 
-    LARGE_FILE=1024*1024*5
-    MULTIPART_SPLIT=10000000
+    LARGE_FILE = 1024 * 1024 * 5
+    MULTIPART_SPLIT = 10000000
 
     attr_reader :base_resource
 
@@ -33,7 +33,7 @@ module Cnvrg
 
     def check_file_sha1(filename, org_sha1, tag: 'conflict')
       file_loc = "#{Dir.pwd}/#{filename}"
-      sha1 =  OpenSSL::Digest::SHA1.file(file_loc).hexdigest
+      sha1 = OpenSSL::Digest::SHA1.file(file_loc).hexdigest
       return 0 if sha1 == org_sha1
       FileUtils.cp(file_loc, "#{file_loc}.#{tag}")
       return 1
@@ -52,7 +52,7 @@ module Cnvrg
         end
         raise SignalException.new(1, "Cant find file #{file}") unless File.exists? "#{Dir.pwd}/#{file}"
       end
-      paths.map{|p| p.gsub(/^\.\//, '')}
+      paths.map { |p| p.gsub(/^\.\//, '') }
     end
 
     def check_files_sha1(files, resolver, tag)
@@ -83,11 +83,11 @@ module Cnvrg
       prefix = @dataset.get_dataset["bucket_prefix"]
       batch_size = 10000
       pbar = ProgressBar.create(:title => "Download Progress",
-                                        :progress_mark => '=',
-                                        :format => "%b%i| %c Files downloaded",
-                                        :starting_at => 0,
-                                        :total => nil,
-                                        :autofinish => true)
+                                :progress_mark => '=',
+                                :format => "%b%i| %c Files downloaded",
+                                :starting_at => 0,
+                                :total => nil,
+                                :autofinish => true)
       parallel_options = {
           in_threads: ParallelThreads,
           in_processes: Cnvrg::CLI::ParallelProcesses,
@@ -116,7 +116,7 @@ module Cnvrg
       if response.present?
         msg = response['result']
       else
-        msg =  "Cant save changes in the dataset"
+        msg = "Cant save changes in the dataset"
       end
 
       Cnvrg::Result.new(Cnvrg::CLI.is_response_success(response, false), msg)
@@ -171,9 +171,9 @@ module Cnvrg
           end
           @temp_upload_progressbar.progress += 1 if @temp_upload_progressbar.present?
         end
-        blob_ids = files.values.map {|f| f['bv_id']}
+        blob_ids = files.values.map { |f| f['bv_id'] }
         if blob_ids.present?
-          dirs = tree.keys.select {|k| tree[k].nil?} || []
+          dirs = tree.keys.select { |k| tree[k].nil? } || []
           Cnvrg::Logger.info("Sending Upload files save")
           upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
           unless Cnvrg::CLI.is_response_success(upload_resp, false)
@@ -208,12 +208,12 @@ module Cnvrg
       file_size = File.size(absolute_path).to_f
       mime_type = MimeMagic.by_path(absolute_path)
       content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
-      sha1 =  OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
+      sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
       if (absolute_path.include? "_tags.yml" or absolute_path.include? "_tags.yaml")
         is_valid = false
         begin
-        content = open(absolute_path).read()
-        hash = YAML.load(open(absolute_path).read())
+          content = open(absolute_path).read()
+          hash = YAML.load(open(absolute_path).read())
           # if level 1 keys count is 1
           if hash.keys.count == 1
             if hash["tags"].present?
@@ -221,7 +221,7 @@ module Cnvrg
             elsif hash[hash.keys.first].class != Hash
               is_valid = true
             end
-          # if level 1 keys count is greater than 1
+            # if level 1 keys count is greater than 1
           elsif hash.keys.count > 1
             if hash["tags"].present? and hash["tags"].class == Hash
               is_valid = false
@@ -255,6 +255,7 @@ module Cnvrg
         return s3_res
       end
     end
+
     def upload_tar_file(absolute_path, relative_path, commit_sha1)
       begin
         file_name = File.basename relative_path
@@ -262,24 +263,24 @@ module Cnvrg
         mime_type = MimeMagic.by_path(absolute_path)
         content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
         begin
-          chunked_bytes = [100, (file_size*0.01)].min
+          chunked_bytes = [100, (file_size * 0.01)].min
           total_yanked = ""
           open(absolute_path, "rb") do |f|
             total_yanked = f.read(chunked_bytes)
           end
           if !total_yanked.empty?
-            sha1 =  OpenSSL::Digest::SHA1.hexdigest(total_yanked)
+            sha1 = OpenSSL::Digest::SHA1.hexdigest(total_yanked)
           else
-            sha1 =  OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
+            sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
           end
         rescue
-          sha1 =  OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
+          sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
         end
 
         upload_resp = Cnvrg::API.request(@base_resource + "upload_tar_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
                                                                                            commit_sha1: commit_sha1, file_name: file_name,
                                                                                            file_size: file_size, file_content_type: content_type, sha1: sha1,
-                                                                                            new_version:true})
+                                                                                           new_version: true})
         if Cnvrg::CLI.is_response_success(upload_resp, false)
           path = upload_resp["result"]["path"]
           s3_res = upload_large_files_s3(upload_resp, absolute_path)
@@ -315,14 +316,15 @@ module Cnvrg
       return false
 
     end
-    def upload_data_log_file(absolute_path, relative_path,data_commit_sha)
+
+    def upload_data_log_file(absolute_path, relative_path, data_commit_sha)
       file_name = File.basename relative_path
       file_size = File.size(absolute_path).to_f
       content_type = "text/x-log"
       upload_resp = Cnvrg::API.request("/users/#{@owner}/" + "upload_data_log", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
-                                                                                             file_name: file_name, log_date: Time.now,
-                                                                                             file_size: file_size, file_content_type: content_type,
-                                                                                              data_commit_sha1:data_commit_sha})
+                                                                                              file_name: file_name, log_date: Time.now,
+                                                                                              file_size: file_size, file_content_type: content_type,
+                                                                                              data_commit_sha1: data_commit_sha})
       if Cnvrg::CLI.is_response_success(upload_resp, false)
         path = upload_resp["result"]["path"]
         s3_res = upload_small_files_s3(path, absolute_path, "text/plain")
@@ -413,15 +415,15 @@ module Cnvrg
 
     end
 
-    def upload_cnvrg_image(absolute_path, image_name,secret)
+    def upload_cnvrg_image(absolute_path, image_name, secret)
       file_name = File.basename relative_path
       file_size = File.size(absolute_path).to_f
       mime_type = MimeMagic.by_path(absolute_path)
       content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
       upload_resp = Cnvrg::API.request("images/#{image_name}/upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: absolute_path,
-                                                                                     file_name: file_name,
-                                                                                     file_size: file_size, file_content_type: content_type,
-                                                                                     secret: secret})
+                                                                                         file_name: file_name,
+                                                                                         file_size: file_size, file_content_type: content_type,
+                                                                                         secret: secret})
       if Cnvrg::CLI.is_response_success(upload_resp, false)
         path = upload_resp["result"]["path"]
         s3_res = upload_large_files_s3(upload_resp, absolute_path)
@@ -465,7 +467,7 @@ module Cnvrg
         sts_path = upload_resp["result"]["path_sts"]
 
         retries = 0
-        success= false
+        success = false
         while !success and retries < 20
           begin
             if !Helpers.is_verify_ssl
@@ -475,7 +477,7 @@ module Cnvrg
             end
             success = true
           rescue => e
-            retries +=1
+            retries += 1
             sleep(5)
 
           end
@@ -487,33 +489,33 @@ module Cnvrg
         key = split[0]
         iv = split[1]
         is_s3 = upload_resp["result"]["is_s3"]
-        access =  Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_a"])
+        access = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_a"])
 
-        secret =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["sts_s"])
+        secret = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_s"])
 
-        session =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["sts_st"])
-        region =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["region"])
+        session = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_st"])
+        region = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["region"])
 
-        bucket =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["bucket"])
-        server_side_encryption =upload_resp["result"]["server_side_encryption"]
+        bucket = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["bucket"])
+        server_side_encryption = upload_resp["result"]["server_side_encryption"]
         use_accelerate_endpoint = false
         if is_s3 or is_s3.nil?
 
-          use_accelerate_endpoint =true
+          use_accelerate_endpoint = true
           client = Aws::S3::Client.new(
-              :access_key_id =>access,
+              :access_key_id => access,
               :secret_access_key => secret,
               :session_token => session,
               :region => region,
               :http_open_timeout => 60, :retry_limit => 20)
         else
-          endpoint = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["endpoint"])
+          endpoint = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["endpoint"])
           use_accelerate_endpoint = false
           client = Aws::S3::Client.new(
-              :access_key_id =>access,
+              :access_key_id => access,
               :secret_access_key => secret,
               :region => region,
-              :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
+              :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
               :http_open_timeout => 60, :retry_limit => 20)
         end
 
@@ -525,13 +527,11 @@ module Cnvrg
         end
 
 
-
-
         s3 = Aws::S3::Resource.new(client: client)
 
         resp = s3.bucket(bucket).
-            object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
-            upload_file(file_path,options)
+            object(upload_resp["result"]["path"] + "/" + File.basename(file_path)).
+            upload_file(file_path, options)
 
 
         return resp
@@ -588,6 +588,7 @@ module Cnvrg
       response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
       return Cnvrg::CLI.is_response_success(response, false)
     end
+
     def download_list_files_in_query(response, dataset_home)
       sts_path = response["path_sts"]
       if !Helpers.is_verify_ssl
@@ -599,29 +600,29 @@ module Cnvrg
       key = split[0]
       iv = split[1]
 
-      access =  Cnvrg::Helpers.decrypt(key, iv, response["sts_a"])
+      access = Cnvrg::Helpers.decrypt(key, iv, response["sts_a"])
 
-      secret =  Cnvrg::Helpers.decrypt(key,iv, response["sts_s"])
+      secret = Cnvrg::Helpers.decrypt(key, iv, response["sts_s"])
 
-      session =  Cnvrg::Helpers.decrypt(key,iv, response["sts_st"])
-      region =  Cnvrg::Helpers.decrypt(key,iv, response["region"])
+      session = Cnvrg::Helpers.decrypt(key, iv, response["sts_st"])
+      region = Cnvrg::Helpers.decrypt(key, iv, response["region"])
 
-      bucket =  Cnvrg::Helpers.decrypt(key,iv, response["bucket"])
+      bucket = Cnvrg::Helpers.decrypt(key, iv, response["bucket"])
       is_s3 = response["is_s3"]
       if is_s3 or is_s3.nil?
         client = Aws::S3::Client.new(
-            :access_key_id =>access,
+            :access_key_id => access,
             :secret_access_key => secret,
             :session_token => session,
             :region => region,
             :http_open_timeout => 60, :retry_limit => 20)
       else
-        endpoint = Cnvrg::Helpers.decrypt(key,iv, response["endpoint_url"])
+        endpoint = Cnvrg::Helpers.decrypt(key, iv, response["endpoint_url"])
         client = Aws::S3::Client.new(
-            :access_key_id =>access,
+            :access_key_id => access,
             :secret_access_key => secret,
             :region => region,
-            :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
+            :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
             :http_open_timeout => 60, :retry_limit => 20)
       end
       list_files = response["files"]
@@ -638,18 +639,18 @@ module Cnvrg
       }
       download_count = 0
       Parallel.map((list_files), parallel_options) do |f|
-        file_key =  Cnvrg::Helpers.decrypt(key,iv, f["path"])
+        file_key = Cnvrg::Helpers.decrypt(key, iv, f["path"])
         begin
-        begin
-          dir = File.dirname f["fullpath"]
-          FileUtils.mkdir_p(dataset_home+"/"+ dir) unless File.exist? (dataset_home+"/"+ dir)
-        end
+          begin
+            dir = File.dirname f["fullpath"]
+            FileUtils.mkdir_p(dataset_home + "/" + dir) unless File.exist? (dataset_home + "/" + dir)
+          end
 
-        File.open(dataset_home+"/"+f["fullpath"], 'w+') do |file|
-          resp = client.get_object({bucket:bucket,
-                                    key:file_key}, target: file)
-        end
-        download_count += 1
+          File.open(dataset_home + "/" + f["fullpath"], 'w+') do |file|
+            resp = client.get_object({bucket: bucket,
+                                      key: file_key}, target: file)
+          end
+          download_count += 1
         rescue
         end
 
@@ -661,97 +662,95 @@ module Cnvrg
       end
 
 
-
-
     end
 
 
-    def download_file_s3(absolute_path, relative_path, project_home, conflict=false, commit_sha1=nil, as_link=false)
+    def download_file_s3(absolute_path, relative_path, project_home, conflict = false, commit_sha1 = nil, as_link = false)
       begin
-        res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1 ,new_version:true, as_link:as_link})
+        res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1, new_version: true, as_link: as_link})
         Cnvrg::CLI.is_response_success(res, false)
-          if res["result"]
-            file_url = res["result"]["file_url"]
+        if res["result"]
+          file_url = res["result"]["file_url"]
 
-            if as_link
-              return res["result"]
-            end
-            # begin
-            #   if !Helpers.is_verify_ssl
-            #     tempfile = Down.download(file_url,open_timeout: 60,ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE )
-            #
-            #   else
-            #     tempfile = Down.download(file_url,open_timeout: 60)
-            #
-            #   end
-            #
-            #   FileUtils.move(tempfile.path, project_home+"/"+ absolute_path)
-            #   return true
-            # rescue
-            #
-            # end
-            download_resp = res
-            filename = download_resp["result"]["filename"]
-
-            absolute_path += ".conflict" if conflict
-            sts_path = download_resp["result"]["path_sts"]
-            retries = 0
-            success= false
-            while !success and retries < 20
-              begin
-                if !Helpers.is_verify_ssl
-                  body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
-                else
-                  body = open(sts_path).read
-                end
-                success = true
-              rescue => e
-                retries +=1
-                sleep(5)
+          if as_link
+            return res["result"]
+          end
+          # begin
+          #   if !Helpers.is_verify_ssl
+          #     tempfile = Down.download(file_url,open_timeout: 60,ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE )
+          #
+          #   else
+          #     tempfile = Down.download(file_url,open_timeout: 60)
+          #
+          #   end
+          #
+          #   FileUtils.move(tempfile.path, project_home+"/"+ absolute_path)
+          #   return true
+          # rescue
+          #
+          # end
+          download_resp = res
+          filename = download_resp["result"]["filename"]
 
+          absolute_path += ".conflict" if conflict
+          sts_path = download_resp["result"]["path_sts"]
+          retries = 0
+          success = false
+          while !success and retries < 20
+            begin
+              if !Helpers.is_verify_ssl
+                body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
+              else
+                body = open(sts_path).read
               end
+              success = true
+            rescue => e
+              retries += 1
+              sleep(5)
+
             end
-            if !success
-              return false
-            end
-            split = body.split("\n")
-            key = split[0]
-            iv = split[1]
+          end
+          if !success
+            return false
+          end
+          split = body.split("\n")
+          key = split[0]
+          iv = split[1]
 
-            access =  Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
+          access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
 
-            secret =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_s"])
+          secret = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_s"])
 
-            session =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_st"])
-            region =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
+          session = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_st"])
+          region = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["region"])
 
-            bucket =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
-            file_key =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
+          bucket = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["bucket"])
+          file_key = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["key"])
 
-            is_s3 = download_resp["result"]["is_s3"]
-            if is_s3 or is_s3.nil?
-              client = Aws::S3::Client.new(
-                  :access_key_id =>access,
-                  :secret_access_key => secret,
-                  :session_token => session,
-                  :region => region,
-                  :http_open_timeout => 60, :retry_limit => 20)
-            else
-              endpoint = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["endpoint_url"])
-              client = Aws::S3::Client.new(
-                  :access_key_id =>access,
-                  :secret_access_key => secret,
-                  :region => region,
-                  :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
-                  :http_open_timeout => 60, :retry_limit => 20)
-            end
+          is_s3 = download_resp["result"]["is_s3"]
+          if is_s3 or is_s3.nil?
+            client = Aws::S3::Client.new(
+                :access_key_id => access,
+                :secret_access_key => secret,
+                :session_token => session,
+                :region => region,
+                :http_open_timeout => 60, :retry_limit => 20)
+          else
+            endpoint = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["endpoint_url"])
+            client = Aws::S3::Client.new(
+                :access_key_id => access,
+                :secret_access_key => secret,
+                :region => region,
+                :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
+                :http_open_timeout => 60, :retry_limit => 20)
+          end
 
-            File.open(project_home+"/"+absolute_path, 'w+') do |file|
-              resp = client.get_object({bucket:bucket,
-                                        key:file_key}, target: file)
-            end
-            return true
+          File.open(project_home + "/" + absolute_path, 'w+') do |file|
+            resp = client.get_object({bucket: bucket,
+                                      key: file_key}, target: file)
           end
+          return true
+        end
 
       rescue => e
         return false
@@ -761,7 +760,7 @@ module Cnvrg
 
     def download_data_file(commit_sha1, dataset_home)
       begin
-        res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1: commit_sha1,new_version:true})
+        res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1: commit_sha1, new_version: true})
         Cnvrg::CLI.is_response_success(res, false)
         if res["result"]
           download_resp = res
@@ -778,38 +777,38 @@ module Cnvrg
           iv = split[1]
 
 
-          access =  Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
+          access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
 
-          secret =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_s"])
+          secret = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_s"])
 
-          session =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_st"])
-          region =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
+          session = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_st"])
+          region = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["region"])
 
-          bucket =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
-          file_key =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
+          bucket = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["bucket"])
+          file_key = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["key"])
 
           is_s3 = download_resp["result"]["is_s3"]
           if is_s3 or is_s3.nil?
             client = Aws::S3::Client.new(
-                :access_key_id =>access,
+                :access_key_id => access,
                 :secret_access_key => secret,
                 :session_token => session,
                 :region => region,
                 :http_open_timeout => 60, :retry_limit => 20)
           else
-            endpoint = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["endpoint_url"])
+            endpoint = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["endpoint_url"])
             client = Aws::S3::Client.new(
-                :access_key_id =>access,
+                :access_key_id => access,
                 :secret_access_key => secret,
                 :region => region,
-                :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
+                :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
                 :http_open_timeout => 60, :retry_limit => 20)
           end
 
 
-          File.open(dataset_home+"/"+filename, 'w+') do |file|
+          File.open(dataset_home + "/" + filename, 'w+') do |file|
             resp = client.get_object({bucket: bucket,
-                                  key: file_key }, target: file)
+                                      key: file_key}, target: file)
           end
           return filename
         end
@@ -820,7 +819,7 @@ module Cnvrg
       end
     end
 
-    def download_file(absolute_path, relative_path, project_home, conflict=false)
+    def download_file(absolute_path, relative_path, project_home, conflict = false)
       res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
       Cnvrg::CLI.is_response_success(res, false)
       if res["result"]
@@ -858,7 +857,7 @@ module Cnvrg
       puts FileUtils.rmtree("#{absolute_path}")
     end
 
-    def revoke_download_file(absolute_path, filename, conflict=false)
+    def revoke_download_file(absolute_path, filename, conflict = false)
       begin
         file_location = absolute_path.gsub(/#{filename}\/?$/, "")
 
@@ -869,6 +868,7 @@ module Cnvrg
         return false
       end
     end
+
     def revoke_download(tar_files, extracted_files)
       begin
 
@@ -882,32 +882,35 @@ module Cnvrg
       return true
 
     end
+
     def delete_commit(commit_sha1)
       response = Cnvrg::API.request("#{base_resource}/commit/#{commit_sha1}", 'DELETE')
       Cnvrg::CLI.is_response_success(response, true)
       return response
     end
+
     def get_commit(commit_sha1)
       response = Cnvrg::API.request("#{base_resource}/commit/#{commit_sha1}", 'GET')
       Cnvrg::CLI.is_response_success(response, true)
       return response
     end
 
-    def start_commit(new_branch,force=false,delete_commit=nil, chunks: 0, dataset: @dataset, message:nil)
+    def start_commit(new_branch, force = false, delete_commit = nil, chunks: 0, dataset: @dataset, message: nil)
       begin
         #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
         # create a new commit.
         idx = (force || new_branch) ? {} : dataset.get_idx
         commit = idx[:commit]
         next_commit = idx[:next_commit]
-        response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
-                                                                                username: @owner,current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
+        response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch, force: force,
+                                                                                username: @owner, current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
         Cnvrg::CLI.is_response_success(response, true)
         return response
       rescue => e
         return false
       end
     end
+
     def last_valid_commit()
       begin
         #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
@@ -920,9 +923,9 @@ module Cnvrg
       end
     end
 
-    def end_commit(commit_sha1,force, success: true, uploaded_files: 0 )
+    def end_commit(commit_sha1, force, success: true, uploaded_files: 0)
       begin
-        response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force, success: success, uploaded_files: uploaded_files})
+        response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1, force: force, success: success, uploaded_files: uploaded_files})
         Cnvrg::CLI.is_response_success(response, true)
         return response
       rescue => e
@@ -948,13 +951,13 @@ module Cnvrg
     end
 
     def get_trees(commit: "latest")
-      response = Cnvrg::API.request("#{@base_resource}/clone_trees", 'POST',{commit: commit})
+      response = Cnvrg::API.request("#{@base_resource}/clone_trees", 'POST', {commit: commit})
       return nil unless Cnvrg::CLI.is_response_success(response, false)
       response['result']['files']
     end
 
     def get_clone_chunk(latest_id: nil, chunk_size: 1000, commit: 'latest')
-      response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST',{commit: commit, chunk_size: chunk_size, latest_id: latest_id})
+      response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST', {commit: commit, chunk_size: chunk_size, latest_id: latest_id})
       unless Cnvrg::CLI.is_response_success(response, false)
         Cnvrg::Logger.log_info("#{{commit: commit, chunk_size: chunk_size, latest_id: latest_id}}")
         return nil
@@ -985,26 +988,26 @@ module Cnvrg
       end
     end
 
-    def download_multiple_chunks(commit, chunk_size=1000, progress: nil)
+    def download_multiple_chunks(commit, chunk_size = 1000, progress: nil)
       begin
-      last_chunk_size = chunk_size
-      q = { commit: commit, chunk_size: chunk_size}
-      overall = 0
-      while last_chunk_size > 0
-        response = Cnvrg::API.request("users/#{@owner}/datasets/#{@dataset_slug}/clone", 'POST', q)
-        if Cnvrg::CLI.is_response_success(response, false) 
-          files = response['files']
-          data_home = "#{Dir.pwd}/#{response['name']}"
-          last_chunk_size = files['keys'].length
-          break if last_chunk_size == 0
-          res = download_multiple_files_s3(files, data_home, read_only: false, progressbar: progress)
-          overall += last_chunk_size
-          q[:latest] = files['latest']
-        else
-          last_chunk_size = 0
+        last_chunk_size = chunk_size
+        q = {commit: commit, chunk_size: chunk_size}
+        overall = 0
+        while last_chunk_size > 0
+          response = Cnvrg::API.request("users/#{@owner}/datasets/#{@dataset_slug}/clone", 'POST', q)
+          if Cnvrg::CLI.is_response_success(response, false)
+            files = response['files']
+            data_home = "#{Dir.pwd}/#{response['name']}"
+            last_chunk_size = files['keys'].length
+            break if last_chunk_size == 0
+            res = download_multiple_files_s3(files, data_home, read_only: false, progressbar: progress)
+            overall += last_chunk_size
+            q[:latest] = files['latest']
+          else
+            last_chunk_size = 0
+          end
         end
-      end
-      Cnvrg::Result.new(true, "Cloned #{overall} files!")
+        Cnvrg::Result.new(true, "Cloned #{overall} files!")
       rescue => e
         Cnvrg::Result.new(false, "Cant download chunk", e.message, e.backtrace)
 
@@ -1012,10 +1015,10 @@ module Cnvrg
     end
 
     def generate_parallel_idx
-      
+
     end
 
-    def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false)
+    def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only: false)
       begin
         refresh_storage_token
         parallel_options = {
@@ -1024,31 +1027,31 @@ module Cnvrg
         }
         Parallel.map(files["keys"], parallel_options) do |f|
           begin
-          local_path = @dataset.local_path + '/' + f['name']
-          Cnvrg::Logger.log_info("Downloading #{local_path}")
-          progressbar.progress += 1 if progressbar.present?
-          if local_path.end_with? "/"
-            @downloader.mkdir(local_path, recursive: true)
-            next
-          end
+            local_path = @dataset.local_path + '/' + f['name']
+            Cnvrg::Logger.log_info("Downloading #{local_path}")
+            progressbar.progress += 1 if progressbar.present?
+            if local_path.end_with? "/"
+              @downloader.mkdir(local_path, recursive: true)
+              next
+            end
             # blob
-          local_path = "#{local_path}.conflict" if conflict
-          storage_path = f["path"]
-          if File.exists? local_path
-            Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
-            next
-          end
-          resp = @downloader.download(storage_path, local_path)
-          Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
+            local_path = "#{local_path}.conflict" if conflict
+            storage_path = f["path"]
+            if File.exists? local_path
+              Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
+              next
+            end
+            resp = @downloader.download(storage_path, local_path)
+            Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
           rescue => e
             Cnvrg::Logger.log_error(e)
           end
         end
-        return Cnvrg::Result.new(true,"Downloaded successfully")
+        return Cnvrg::Result.new(true, "Downloaded successfully")
       rescue => e
-          Cnvrg::Logger.log_error(e)
-          return Cnvrg::Result.new(false,"Could not download some files", e.message, e.backtrace)
-        end
+        Cnvrg::Logger.log_error(e)
+        return Cnvrg::Result.new(false, "Could not download some files", e.message, e.backtrace)
       end
+    end
   end
 end

data/lib/cnvrg/project.rb

@@ -370,6 +370,13 @@ module Cnvrg
       File.open(@working_dir + "/.cnvrg/config.yml", "w+") {|f| f.write new_config.to_yaml}
     end
 
+    def generate_git_diff
+      git_diff = `git diff --name-only`
+      git_diff.split("\n")
+    rescue
+      []
+    end
+
     def generate_output_dir(output_dir)
       Cnvrg::Logger.log_info("Generating output dir for #{output_dir}")
       upload_list = []
@@ -387,7 +394,7 @@ module Cnvrg
         upload_list << output_dir + "/"
       end
       Cnvrg::Logger.log_info("Uploading: #{upload_list.join(", ")}")
-      return upload_list
+      upload_list
     end
     def generate_output_dir_tmp(output_dir)
       upload_list = []

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.6.36'
+  VERSION = '1.6.38'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.6.36
+  version: 1.6.38
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-03 00:00:00.000000000 Z
+date: 2020-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler