cnvrg 0.6.3 → 0.6.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c1eb2a526f32419697a93b9927b01ec1327588db
-  data.tar.gz: cbc3517d2341d6a1905190a6a6f99bfa1e1df20c
+  metadata.gz: 88c66150eee5e836d00d2cbf4983ab978b227f6f
+  data.tar.gz: 28ff50dfd3e535e640ebd5d7a2333dc55f954176
 SHA512:
-  metadata.gz: 9287cf89d3440a871737dd037c2bc5b3dc6ebc2457ea31cfbd1156d022891460977d44a5993f06620d396ec9604ce5786ba63e957672773dac32a259a2792b45
-  data.tar.gz: 66a9c63f4adccb2720d9cbee9b7f17ac79cedda0e9389bd9300b830c02292ad03259eec1c42e427ea095df1bffe1fa5496d1239d96fe37ce5b207041def87b82
+  metadata.gz: d5276cacdecc9b1860e1f1a966d93186b9c5ff00320f53d65cf4bed39ac67ff4ba927a73d232cd98fd3d54a3deb237dde92289079d648583bf30a7672ffdf975
+  data.tar.gz: 9c84dca4965fc101de56e573fbd655e290c34c068c9d0598b524a82436ddb332af02228407e8c7c517734a992ad67ba489b64d2d35092deb248ce59cdff03455

data/lib/cnvrg/cli.rb

@@ -1,5 +1,6 @@
 #!/usr/bin/env ruby
-require "pty"
+require "pty" unless !!(RUBY_PLATFORM =~ /mswin32|mingw32/)
+
 require 'etc'
 require 'parallel'
 require 'netrc'
@@ -844,6 +845,7 @@ module Cnvrg
                                            :starting_at => 0,
                                            :total => files_count,
                                            :autofinish => true)
+          
           while files['keys'].length > 0
             @files.download_multiple_files_s3(files, dataset_home, progressbar: progressbar)
             downloaded_files += files['keys'].length
@@ -1922,8 +1924,10 @@ module Cnvrg
       log_start(__method__, args, options)
       log_message('Syncing dataset', Thor::Shell::Color::BLUE, !options["verbose"])
       if !force and !init
-        invoke :download_data_new,[verbose,true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
+        # w(verbose=false, new_branch=false,sync=false, commit=nil,all_files=true)
+        invoke :download_data_new,[verbose, new_branch, true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
       end
+      # w(new_branch,  verbose,sync,force, tags, chunk_size)
       invoke :upload_data_new,[new_branch,  verbose,true,force, tags, chunk_size], :new_branch=>new_branch,
              :direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel
 
@@ -1936,6 +1940,7 @@ module Cnvrg
     method_option :tags, :type => :boolean, :aliases => ["--tags"], :desc => "upload file tags", :default => false
     method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
     def upload_data_new(new_branch,  verbose,sync,force, tags, chunk_size)
+      begin
       commit = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size
       upload_res = invoke :upload_data_files,[commit],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
       if upload_res < 0
@@ -1962,11 +1967,21 @@ module Cnvrg
           log_message('There was some error in uploading Tags', Thor::Shell::Color::RED)
         end
       end
-
-      if upload_res
-        invoke :end_commit_data,[commit] , :new_branch=>new_branch,  :force =>force
-      end
-
+    rescue => e
+      say "\nAborting", Thor::Shell::Color::BLUE
+      dataset_dir = is_cnvrg_dir(Dir.pwd)
+      return false if dataset_dir.blank?
+      @dataset = Dataset.new(dataset_dir)
+      @dataset.revert_to_last_commit
+      return false
+    rescue SignalException
+      say "\nAborting", Thor::Shell::Color::BLUE
+      dataset_dir = is_cnvrg_dir(Dir.pwd)
+      return false if dataset_dir.blank?
+      @dataset = Dataset.new(dataset_dir)
+      @dataset.revert_to_last_commit
+      return false
+    end
     end
 
     desc 'start_commit', 'start data commit'
@@ -2014,18 +2029,7 @@ module Cnvrg
           puts commit_sha1
         end
         return commit_sha1
-      rescue => e
-          if commit_sha1.present?
-            @dataset.set_next_commit(commit_sha1)
-          end
-      rescue SignalException
-        log_message("Aborting..", Thor::Shell::Color::YELLOW)
-        if commit_sha1.present?
-          @dataset.set_next_commit(commit_sha1)
-        end
-        exit(1)
       end
-
     end
     desc 'end_commit', 'start data commit'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
@@ -2049,18 +2053,9 @@ module Cnvrg
             log_message("#{check} Data files were updated successfully", Thor::Shell::Color::GREEN)
             @dataset.set_next_commit(resp["result"]["commit_sha1"]) # no need fo rboth methods FIX
             @dataset.remove_next_commit()
-
           end
-
         end
-      rescue => e
-        puts e
-      rescue SignalException
-        @dataset.set_next_commit(commit)
-        log_message("Aborting", Thor::Shell::Color::YELLOW)
-        exit(1)
       end
-
     end
     desc 'list_files', 'list files in dataset'
     method_option :json, :type => :boolean, :aliases => ["-j","--json"],:default => true,   :desc => "response as json"
@@ -2137,12 +2132,13 @@ module Cnvrg
         files_uploaded = 0
         all_files.each_slice(chunk_size).each do |list_files|
           temp_tree = @dataset.generate_chunked_idx(list_files, threads: parallel_threads)
+
           upload_resp = @files.upload_multiple_files(new_commit, temp_tree, threads: parallel_threads, force: force, new_branch: new_branch, progressbar: progressbar)
 
           if upload_resp.is_a? Cnvrg::Result
             log_message(upload_resp.msg, upload_resp.msg_color)
-
-            return -1
+            raise SignalException.new(upload_resp.msg)
+            # return -1
 
           else
             files_uploaded += upload_resp
@@ -2155,9 +2151,6 @@ module Cnvrg
           @dataset.write_idx(new_tree, new_commit)
         end
         return files_uploaded
-      rescue Exception => e
-        puts e
-        return false
       end
 
       #   local_idx = @dataset.generate_idx
@@ -2522,7 +2515,6 @@ module Cnvrg
             rescue SignalException
               log_message("Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED)
               @files.rollback_commit(commit_sha1)
-
               return false
             end
           end
@@ -2767,30 +2759,30 @@ module Cnvrg
         dataset_dir = is_cnvrg_dir(Dir.pwd)
         @dataset = Dataset.new(dataset_dir)
         @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
-        commit_to_download = commit
-        local_idx = @dataset.generate_idx
         all_files = all_files
-        res = @dataset.compare_idx_download(commit=commit_to_download.presence, local_idx= local_idx,all_files=all_files)
+        res = @dataset.compare_idx_download(all_files: all_files, desired_commit: commit)
+        unless CLI.is_response_success(res, false)
+          log_message("Cant find the desired commit, please check it or try to download without it.", Thor::Shell::Color::RED)
+          exit(1)
+        end
         result = res["result"]["tree"]
         commit = res["result"]["commit"]
-        if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
-          log_message("Dataset is up to date", Thor::Shell::Color::GREEN, ((options["sync"] or sync) ? false : true))
-          return true
-        end
-        update_total = result["updated_on_server"].size + result["conflicts"].size + result["deleted"].size
-        successful_changes = 0
-        if update_total ==1
-          log_message("Downloading #{update_total} file", Thor::Shell::Color::BLUE, !sync)
-        elsif update_total == 0
-          log_message("Dataset is up to date", Thor::Shell::Color::GREEN, !sync)
+        update_total = [result['added'], result["updated_on_server"], result["conflicts"], result["deleted"]].compact.flatten.size
+        successful_changes  = 0
+        if update_total == 0
+          log_message("Dataset is up to date", Thor::Shell::Color::GREEN) if !sync
           return true
         elsif options["verbose"]
           log_message("Downloading #{update_total} files", Thor::Shell::Color::BLUE)
         else
           log_message("Syncing Dataset", Thor::Shell::Color::BLUE, !sync)
         end
+        conflicts = @files.mark_conflicts(result)
+        
+        log_message("Found some conflicts, check .conflict files.", Thor::Shell::Color::BLUE) if conflicts > 0
         update_res = @files.download_files_in_chunks(result["updated_on_server"]) if result["updated_on_server"].present?
-        conflict_res = @files.download_files_in_chunks(result["conflicts"], conflict: true) if result["conflicts"].present?
+        added_res = @files.download_files_in_chunks(result["added"]) if result["added"].present?
+        # conflict_res = @files.download_files_in_chunks(result["conflicts"], conflict: true) if result["conflicts"].present?
         deleted = result["deleted"].to_a
         delete_res = @files.delete_commit_files_local(deleted)
         if !delete_res
@@ -2798,7 +2790,10 @@ module Cnvrg
           log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
           exit(1)
         end
-        if [update_res.is_success?, delete_res.is_success?, (conflict_res.blank? or conflict_res.is_success?)].all?
+        success = (update_res.blank? or update_res.is_success?)
+        success &= (delete_res.blank? or delete_res.is_success?)
+        success &= (added_res.blank? or added_res.is_success?)
+        if success
           # update idx with latest commit
           @dataset.update_idx_with_commit!(commit)
           check = Helpers.checkmark()
@@ -2809,17 +2804,15 @@ module Cnvrg
           else
             log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, ((sync or options["sync"]) ? false : true))
           end
-          return true
-        else
-          @dataset.update_idx_with_commit!(commit_to_download)
+          return true          
         end
+      rescue SignalException
+        say "\nAborting", Thor::Shell::Color::BLUE
+        exit(1)
       rescue => e
         log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
         log_error(e)
         exit(1)
-      rescue SignalException
-        say "\nAborting", Thor::Shell::Color::BLUE
-        exit(1)
       end
     end
 
@@ -3339,10 +3332,15 @@ module Cnvrg
       options_hash = Hash[options]
 
       if local
-        invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
-               :log => log, :email_notification => email_notification, :upload_output => upload_output,
-               :commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :output_dir=>output_dir, :data_query=>data_query
-        return
+        if Cnvrg::Helpers.windows?
+            say "Windows is currently not supported for running experiments locally"
+            return
+        else
+          invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
+                 :log => log, :email_notification => email_notification, :upload_output => upload_output,
+                 :commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :output_dir=>output_dir, :data_query=>data_query
+          return
+        end
       else
         if !periodic_sync.nil? and !periodic_sync.empty?
           if /^\d{2}$/ === periodic_sync

data/lib/cnvrg/datafiles.rb

@@ -20,6 +20,39 @@ module Cnvrg
       @base_resource = "users/#{owner}/datasets/#{dataset_slug}/"
     end
 
+    def check_file_sha1(filename, org_sha1, tag: 'conflict')
+      file_loc = "#{Dir.pwd}/#{filename}"
+      sha1 =  OpenSSL::Digest::SHA1.file(file_loc).hexdigest
+      return 0 if sha1 == org_sha1
+      FileUtils.cp(file_loc, "#{file_loc}.#{tag}")
+      return 1
+    end
+
+    def check_files_sha1(files, resolver, tag)
+      conflicts = 0
+      files.each do |file|
+        next if file.ends_with? '/'
+        sha1 = resolver.fetch(file, {}).fetch("sha1", nil)
+        conflicts += self.check_file_sha1(file, sha1, tag: tag)
+      end
+      conflicts
+    end
+
+    def mark_conflicts(results)
+      begin
+        updated = results["updated_on_server"]
+        deleted = results["deleted"]
+        resolver = results['resolved_files']
+        overall = 0
+        overall += self.check_files_sha1(updated, resolver, "conflict")
+        overall += self.check_files_sha1(deleted, resolver, "deleted")
+        overall
+      rescue => e
+        0
+      end
+    end
+
+
     def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, progressbar: nil)
       random_file_name = (0...8).map { (65 + rand(26)).chr }.join #needed to create a file for post_file..
       # each file have: {sha1: sha1, file_name: file_name, file_size: file_size, content_type: content_type, absolute_path, relative_path} 
@@ -29,9 +62,9 @@ module Cnvrg
       upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch})
       return Cnvrg::Result.new(false, "Failed to upload files") unless Cnvrg::CLI.is_response_success(upload_resp, false)
       results = upload_resp['result'].with_indifferent_access
-        if results['files'].blank?
-          return 0
-        end
+      if results['files'].blank?
+        return 0
+      end
       props = Cnvrg::Helpers.get_s3_props(results)
       client = props[:client]
       bucket = props[:bucket]
@@ -45,7 +78,7 @@ module Cnvrg
         progressbar.progress += 1 if progressbar.present?
       end
       blob_ids = files.values.map{|f| f['bv_id']}
-      dirs = tree.keys.select{|k| tree[k].nil? }
+      dirs = tree.keys.select{|k| tree[k].nil? } || []
       upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
       return files.keys.length
       rescue => e
@@ -808,15 +841,16 @@ module Cnvrg
       (1..5).each do |i|
         response = Cnvrg::API.request("users/#{@owner}/datasets/#{@dataset_slug}/download_multi", 'POST', {files: files})
         next unless Cnvrg::CLI.is_response_success(response, false) #trying to api request 5 times.
-        files = response['files']
+        files_to_download = response['files']
         data_home = "#{Dir.pwd}/#{response['name']}"
-        res = download_multiple_files_s3(files, data_home, conflict: conflict)
+        res = download_multiple_files_s3(files_to_download, data_home, conflict: conflict)
         next unless res.is_success? #try again..
-        return files['keys'].length
+        return files_to_download['keys'].length
       end
     end
 
     def download_multiple_chunks(commit, chunk_size=1000)
+      begin
       last_chunk_size = chunk_size
       q = { commit: commit, chunk_size: chunk_size}
       overall = 0
@@ -835,6 +869,9 @@ module Cnvrg
         end
       end
       Cnvrg::Result.new(true, "Cloned #{overall} files!")
+      rescue => e
+        Cnvrg::Result.new(false, "Cant download chunk")
+      end
     end
 
     def generate_parallel_idx

data/lib/cnvrg/dataset.rb

@@ -314,6 +314,17 @@ module Cnvrg
       return tree
     end
 
+    def revert_to_last_commit(commit: nil)
+      if commit.blank?
+        resp = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/last_valid_commit", 'GET')
+        if CLI.is_response_success(resp, false)
+         commit = resp['result']['commit_sha1']
+        end
+      end
+      self.update_idx_with_commit(commit) if commit.present?
+      self.revert_next_commit
+    end
+
     def list_all_files
       list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~/^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~/^#{self.local_path}\/\.cnvrgignore/) }
       list_ignore = self.get_ignore_list()
@@ -423,15 +434,13 @@ module Cnvrg
       CLI.is_response_success(response,false)
       return response
     end
-    def compare_idx_download(commit=last_local_commit,local_idx=nil,all=false)
-      if local_idx.nil?
-        local_idx =  self.generate_idx
-      end
+    
+    def compare_idx_download(all_files: false, desired_commit: nil)
+      current_commit = self.last_local_commit
       ignore_list = self.send_ignore_list()
-      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/download_status", 'POST', {idx: local_idx,  current_commit: commit, ignore:ignore_list,all_files:all})
-      CLI.is_response_success(response,false)
-      return response
+      return Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/download_status", 'POST', {current_commit: current_commit, ignore:ignore_list,all_files:all_files, desired_commit: desired_commit.presence})
     end
+    
     def current_status(new_branch)
         commit=last_local_commit
         response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status_current", 'POST', {current_commit: commit,new_branch: new_branch})

data/lib/cnvrg/helpers.rb

@@ -263,7 +263,8 @@ parameters:
             :http_open_timeout => 60, :retry_limit => 20)
         use_accelerate_endpoint = true
       else
-        endpoint = Cnvrg::Helpers.decrypt(key,iv, files["endpoint"])
+
+        endpoint = Cnvrg::Helpers.decrypt(key,iv, files["endpoint_url"])
         client = Aws::S3::Client.new(
             :access_key_id =>access,
             :secret_access_key => secret,

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '0.6.3'
+  VERSION = '0.6.5'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.6.5
 platform: ruby
 authors:
 - Yochay Ettun