cnvrg 0.7.0 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1b627b016030a88e2a88088ea28015400b6343bd
-  data.tar.gz: 9b348a048c3488b05e741fcc201648a229abf755
+  metadata.gz: 11c1d6358ef98bf5b6372fc3ddf3348cefd64c5a
+  data.tar.gz: 661547a796ec6a79cd45bb09c9621f4cfb7bace8
 SHA512:
-  metadata.gz: 2cac1941c4a91d93fa4328b4b7e134853bd22c4631af862e83efe52871cc837a97b12ef0172396b4f9fcd206085820f32c5301f07bdcc1c5023442b86677b484
-  data.tar.gz: c4c99f4e19e94461b36b24046e836dd835f24bddd0611f3244c90f56e68fd156f9c197f5baf9cb8dda98a01e4501dcc03242bfd027cced5cf15269ad69029f5b
+  metadata.gz: cc98c43afdcabb9b433c8472504ef62ad3fe5a0bb2097b020c907acdba984a73debc56848a98e8d34866c3efca8f2b255370d084cc54a50b4c922ecf4e8943f5
+  data.tar.gz: 45aef2a6405f28cba33c69feb7202e82b3845d353fcf54283fd0b769306f861e3cb5120faa5a6e19f0db0ba8b38648d5b2ec14273e0e8cc2d839b79ab03fa1a1

data/lib/cnvrg/api.rb

@@ -79,14 +79,15 @@ module Cnvrg
                 else
                     response
                 end
-                when 'POST'
+                when 'POST', 'PUT'
                 conn.options.timeout = 4200
                 conn.options.open_timeout =4200
                 retries = 0
                 success = false
                 while !success and retries < 20
                     begin
-                        response = conn.post "#{resource}", data
+                        response = conn.post "#{resource}", data if method.eql? 'POST'
+                        response = conn.put "#{resource}", data if method.eql? 'PUT'
                         success = true
 
                     rescue => e

data/lib/cnvrg/cli.rb

@@ -1,6 +1,5 @@
 #!/usr/bin/env ruby
 require "pty" unless !!(RUBY_PLATFORM =~ /mswin32|mingw32/)
-
 require 'etc'
 require 'parallel'
 require 'netrc'
@@ -163,6 +162,27 @@ module Cnvrg
 
     desc "", "", :hide => true
 
+    def test
+      # image_settings = {
+      #     'Image' => "cnvrg:latest",
+      #     'User' => 'ds',
+      #     'Cmd' => '/home/ds/run_ipython.sh',
+      #     'ExposedPorts' => {``
+      #         '80/tcp' => {},
+      #     },
+      #     'HostConfig' => {
+      #         'PortBindings' => {
+      #             '80/tcp' => [
+      #                 {'HostPort' => "7654", 'HostIp' => 'localhost'}
+      #             ],
+      #         },
+      #     },
+      # }
+      # container = Docker::Container.get('b4d64bf83f41')
+      # s = "/leah/1/2/3/4/5"
+      # command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{s}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
+      # puts container.exec(command, tty: true)
+    end
 
 
     desc 'version', 'Prints cnvrg current version'
@@ -172,6 +192,32 @@ module Cnvrg
 
     end
 
+
+    desc 'dataurl', 'dataur', :hide => true
+    def set_data_url(dataset_url)
+      begin
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      unless is_cnvrg_dir
+        log_message("Not in cnvrg dir.", Thor::Shell::Color::RED)
+      end
+      url_parts = dataset_url.split("/")
+      project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
+      slug = url_parts[project_index + 1]
+      owner = url_parts[project_index - 1]
+      res = Cnvrg::API.request("users/#{owner}/datasets/#{slug}", 'GET')
+      unless Cnvrg::CLI.is_response_success(res, false)
+        raise SignalException.new()
+      end
+      @dataset = Dataset.new(Dir.pwd)
+      result = res['result']
+      @dataset.change_url(result.symbolize_keys)
+      log_message("Changed URL Succesfuly to #{dataset_url}", Thor::Shell::Color::GREEN)
+      rescue => e
+        log_message("Cant change the url to the given dataset url", Thor::Shell::Color::RED)
+      end
+    end
+
     map %w(-v --version) => :version
 
     desc 'api', 'set api url, e.g cnvrg --api "https://cnvrg.io/api"'
@@ -1099,6 +1145,59 @@ module Cnvrg
       exit(1)
     end
 
+    desc '', '', :hide => true
+    def data_put(dataset_url, files: [], dir: '', chunk_size: 1000)
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+
+        #find owner and slug in url
+        url_parts = dataset_url.split("/")
+        project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
+        slug = url_parts[project_index + 1]
+        owner = url_parts[project_index - 1]
+
+        @datafiles = Cnvrg::Datafiles.new(owner, slug)
+        @dataset = Dataset.new(Dir.pwd)
+        @files = @datafiles.verify_files_exists(files)
+
+        if @files.blank?
+          raise SignalException.new(1, "Cant find files to upload, exiting.")
+        end
+        log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
+        number_of_chunks = (@files.size.to_f / chunk_size).ceil
+        response = @datafiles.start_commit(false, true, chunks: number_of_chunks)
+        unless response #means we failed in the start commit.
+          raise SignalException.new(1, "Cant put files into server, check the dataset slug")
+        end
+        @commit = response['result']['commit_sha1']
+        #dir shouldnt have starting or ending slash.
+        dir = dir[0..-2] if dir.end_with? '/'
+        dir = dir[1..-1] if dir.start_with? '/'
+
+        progressbar = ProgressBar.create(:title => "Upload Progress",
+                                         :progress_mark => '=',
+                                         :format => "%b>>%i| %p%% %t",
+                                         :starting_at => 0,
+                                         :total => @files.size,
+                                         :autofinish => true)
+
+        @files.each_slice(chunk_size).each do |list_files|
+          temp_tree = @dataset.generate_chunked_idx(list_files, prefix: dir)
+          #will throw a signal exception if something goes wrong.
+          @datafiles.upload_multiple_files(@commit, temp_tree, force: false, progressbar: progressbar, prefix: dir)
+        end
+        res = @datafiles.put_commit(@commit)
+        unless res.is_success?
+          raise SignalException.new(1, res.msg)
+        end
+        log_message("Upload finished", Thor::Shell::Color::GREEN)
+      rescue SignalException => e
+        log_message(e.message, Thor::Shell::Color::RED)
+        return false
+      end
+    end
+
 
 
 
@@ -2082,7 +2181,6 @@ module Cnvrg
     method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
 
     def upload_data_files(new_commit, *files)
-
       begin
         verify_logged_in(true)
         log_start(__method__, args, options)
@@ -2378,8 +2476,6 @@ module Cnvrg
         end
         end
 
-
-
         if ignore.nil? or ignore.empty?
           ignore = ignore_list
         end
@@ -2394,7 +2490,7 @@ module Cnvrg
         if !@project.update_ignore_list(ignore)
           log_message("Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW)
         end
-        new_branch = options["new_branch"] || false
+        new_branch = options["new_branch"] || @project.is_branch
 
         result = @project.compare_idx(new_branch, force: force, deploy: options["deploy"],in_exp:in_exp, specific_files: spec_files_to_upload)
         commit = result["result"]["commit"]
@@ -2408,8 +2504,6 @@ module Cnvrg
           log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE, (options["verbose"]))
         end
         result = result["result"]["tree"]
-
-
         if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
           log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
           return true
@@ -2428,18 +2522,19 @@ module Cnvrg
           log_message("Syncing files", Thor::Shell::Color::BLUE, ((options["sync"] or sync)) ? false : true)
         end
         # Start commit
+        current_commit = nil
+        exp_start_commit = nil
         if in_exp || (job_slug.present? and job_type.present?)
           exp_start_commit = @project.last_local_commit
         else
-          exp_start_commit = nil
+          current_commit = @project.last_local_commit
         end
         job_type = options['job_type'] || job_type
         job_slug = options['job_slug'] || job_slug
         commit_sha1 = @files.start_commit(new_branch, force: force, exp_start_commit:exp_start_commit,
-                                          job_type: job_type, job_slug: job_slug)["result"]["commit_sha1"]
+                                          job_type: job_type, job_slug: job_slug, start_commit: current_commit)["result"]["commit_sha1"]
         # upload / update
         begin
-
           parallel_options = {
               :progress => {
                   :title => "Upload Progress",
@@ -2869,7 +2964,7 @@ module Cnvrg
         if !@project.update_ignore_list(ignore)
           log_message("Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW)
         end
-        new_branch = options["new_branch"] || false
+        new_branch = options["new_branch"] || @project.is_branch || false
         res = @project.compare_idx(new_branch, commit: nil, in_exp: in_exp)["result"]
         result = res["tree"]
 
@@ -3046,82 +3141,82 @@ module Cnvrg
 
 
     desc 'jump', 'jump to specific commit'
-    method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false
     def jump(commit_sha1)
       begin
         verify_logged_in()
         log_start(__method__, args, options)
-        is_remote = options["remote"]
         project_home = get_project_home
         @project = Project.new(project_home)
-        # say "Syncing existing project tree before jumping", Thor::Shell::Color::BLUE
         current_commit = @project.last_local_commit
         if current_commit.eql? commit_sha1
           log_message("Project is already updated", Thor::Shell::Color::GREEN)
           exit(0)
         end
-        @files = Cnvrg::Files.new(@project.owner, @project.slug)
-        response = @project.clone(false, commit_sha1)
-        Cnvrg::CLI.is_response_success(response, true)
-
-        successful_changes = []
-
-        if !response["result"]["tree"].nil?
-          parallel_options = {
-              :progress => {
-                  :title => "Jump Progress",
-                  :progress_mark => '=',
-                  :format => "%b>>%i| %p%% %t",
-                  :starting_at => 0,
-                  :total => response["result"]["tree"].size,
-                  :autofinish => true
-              },
-              in_processes: ParallelProcesses,
-              in_thread: ParallelThreads
-          }
-          commit_sha1 = response["result"]["commit"]
-          idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
-          File.open(project_home + "/.cnvrg/idx.yml", "w+") {|f| f.write idx.to_yaml}
-          if is_remote
-            current_tree = Dir.glob("**/*", File::FNM_DOTMATCH).flatten.reject {|file| file.start_with? '.' or file.eql? "__init__.py" or file.eql? "uwsgi.ini" or file.ends_with? "/." or file.eql? "."}
-          else
-            current_tree = Dir.glob("**/*", File::FNM_DOTMATCH).flatten.reject {|file| file.start_with? '.cnvrg' or file.ends_with? "/." or file.eql? "."}
-          end
-
-          jump_result = Parallel.map(response["result"]["tree"], parallel_options) do |f|
-
-            relative_path = f[0].gsub(/^#{@project.local_path}/, "")
-            log_message("Downloading #{f[0]}", Thor::Shell::Color::BLUE, false)
-            if f[0].end_with? "/"
-              # dir
-              @files.download_dir(f[0], relative_path, project_home)
-
-            else
-              # blob
-              @files.download_file_s3(f[0], relative_path, project_home, commit_sha1 = commit_sha1)
-            end
-          end
-
-
-          successful_changes = jump_result.select {|x| not x.nil?}
-        end
-        response["result"]["tree"].each do |f|
-          if f[0].end_with? "/"
-            current_tree.delete(f[0][0, f[0].size - 1])
-          else
-            current_tree.delete(f[0])
-          end
+        @files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: project_home)
+        resp = @project.jump_idx(destination: commit_sha1)
+        if resp.blank?
+          log_message("Cant find the given commit", Thor::Shell::Color::RED)
+          exit(0)
         end
-
-        FileUtils.rm_rf(current_tree)
-        log_message("Done. Jumped to #{commit_sha1} completed successfully", Thor::Shell::Color::GREEN)
+        compare = resp['result']['compare']
+        resolver = resp['result']['resolver']
+        latest = resp['result']['latest']
+        commit = resp['result']['commit']
+        @files.handle_compare_idx(compare, resolver: resolver)
+        @project.set_on_branch(latest)
+        @project.update_idx_with_commit!(commit)
+        @project.generate_idx
+        log_message("Jumped successfuly!", Thor::Shell::Color::GREEN)
       rescue => e
-        log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
-        log_error(e)
-      rescue SignalException
-
-        exit(1)
-      end
+        log_message("Cant jump to the specified commit", Thor::Shell::Color::RED)
+      end
+      #   successful_changes = []
+      #
+      #   if !response["result"]["tree"].nil?
+
+      #     commit_sha1 = response["result"]["commit"]
+      #     idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
+      #     File.open(project_home + "/.cnvrg/idx.yml", "w+") {|f| f.write idx.to_yaml}
+      #     if is_remote
+      #       current_tree = Dir.glob("**/*", File::FNM_DOTMATCH).flatten.reject {|file| file.start_with? '.' or file.eql? "__init__.py" or file.eql? "uwsgi.ini" or file.ends_with? "/." or file.eql? "."}
+      #     else
+      #       current_tree = Dir.glob("**/*", File::FNM_DOTMATCH).flatten.reject {|file| file.start_with? '.cnvrg' or file.ends_with? "/." or file.eql? "."}
+      #     end
+      #
+      #     jump_result = Parallel.map(response["result"]["tree"], parallel_options) do |f|
+      #
+      #       relative_path = f[0].gsub(/^#{@project.local_path}/, "")
+      #       log_message("Downloading #{f[0]}", Thor::Shell::Color::BLUE, false)
+      #       if f[0].end_with? "/"
+      #         # dir
+      #         @files.download_dir(f[0], relative_path, project_home)
+      #
+      #       else
+      #         # blob
+      #         @files.download_file_s3(f[0], relative_path, project_home, commit_sha1 = commit_sha1)
+      #       end
+      #     end
+      #
+      #
+      #     successful_changes = jump_result.select {|x| not x.nil?}
+      #   end
+      #   response["result"]["tree"].each do |f|
+      #     if f[0].end_with? "/"
+      #       current_tree.delete(f[0][0, f[0].size - 1])
+      #     else
+      #       current_tree.delete(f[0])
+      #     end
+      #   end
+      #
+      #   FileUtils.rm_rf(current_tree)
+      #   log_message("Done. Jumped to #{commit_sha1} completed successfully", Thor::Shell::Color::GREEN)
+      # rescue => e
+      #   log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
+      #   log_error(e)
+      # rescue SignalException
+      #
+      #   exit(1)
+      # end
     end
 
     desc 'show', 'Show specific file from a specific commit'
@@ -3820,6 +3915,7 @@ module Cnvrg
         if forced_commit and (commit_to_run.nil? or commit_to_run.empty?)
           commit_to_run = forced_commit
         end
+        commit_to_run = commit_to_run.presence || project.last_local_commit
         res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
                               periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,
                               data_query, git_commit, git_branch, restart_if_stuck,local_folders_options )
@@ -5780,6 +5876,22 @@ module Cnvrg
         end
       end
 
+      def self.log_message(message, type = Thor::Shell::Color::BLUE, to_print: true)
+        say "#{type}#{message}" if to_print
+        case type
+        when Thor::Shell::Color::BLUE
+          $LOG.info message: message, type: "info"
+        when Thor::Shell::Color::RED
+          $LOG.error message: message, type: "error"
+        when Thor::Shell::Color::YELLOW
+          $LOG.warn message: message, type: "warning"
+        when Thor::Shell::Color::GREEN
+          $LOG.info message: message, type: "success"
+        else
+          $LOG.info message: message, type: "unknown"
+        end
+      end
+
       def log_message(message, type, to_print = true)
         if to_print
           say message, type

data/lib/cnvrg/colors.rb

@@ -0,0 +1,8 @@
+module Cnvrg
+  class Colors
+    RED = 0
+    BLUE = 1
+    YELLOW = 2
+    GREEN = 3
+  end
+end

data/lib/cnvrg/data.rb

@@ -77,7 +77,6 @@ module Cnvrg
       commit = options["commit"]
       all_files = options["all_files"]
       cli.download_data_new(verbose,sync,new_branch, commit,all_files)
-
     end
     desc 'data clone', 'clone datset'
     method_option :only_tree, :type => :boolean, :aliases => ["-t", "--tree"], :default => false
@@ -92,6 +91,25 @@ module Cnvrg
       cli.clone_data(dataset_url, only_tree=only_tree,commit=commit, query=query)
     end
 
+    desc 'data set --url=dataset_url', 'Set dataset url to other url'
+    method_option :url, :type => :string, :aliases => ["--url"], :default => ''
+    def set
+      cli = Cnvrg::CLI.new
+      cli.log_start(__method__)
+      cli.log_handler
+      if options['url'].present?
+        cli.set_data_url(options['url'])
+      end
+    end
+
+    desc 'date put dataset_url', 'clone datset'
+    method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
+    def put(dataset_url, *files)
+      cli = Cnvrg::CLI.new()
+      dir = options[:dir]
+      cli.data_put(dataset_url, files: files, dir: dir)
+    end
+
     desc 'data clone_query', 'clone query datset'
     method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
     def clone_query(dataset_url)

data/lib/cnvrg/datafiles.rb

@@ -28,6 +28,22 @@ module Cnvrg
       return 1
     end
 
+    def verify_files_exists(files)
+      paths = []
+      files.each do |file|
+        if File.exists? file
+          if File.directory? file
+            paths += Dir.glob("#{file}/**/*")
+          else
+            paths << file
+          end
+          next
+        end
+        raise SignalException.new(1, "Cant find file #{file}") unless File.exists? "#{Dir.pwd}/#{file}"
+      end
+      paths
+    end
+
     def check_files_sha1(files, resolver, tag)
       conflicts = 0
       files.each do |file|
@@ -53,7 +69,19 @@ module Cnvrg
     end
 
 
-    def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, progressbar: nil)
+    def put_commit(commit_sha1)
+      response = Cnvrg::API.request("#{@base_resource}/commit/latest", 'PUT', {commit_sha1: commit_sha1})
+      if response.present?
+        msg = response['result']
+      else
+        msg =  "Cant save changes in the dataset"
+      end
+
+      Cnvrg::Result.new(Cnvrg::CLI.is_response_success(response, false), msg)
+    end
+
+
+    def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, progressbar: nil, prefix: '')
       random_file_name = (0...8).map { (65 + rand(26)).chr }.join #needed to create a file for post_file..
       # each file have: {sha1: sha1, file_name: file_name, file_size: file_size, content_type: content_type, absolute_path, relative_path} 
 
@@ -63,6 +91,7 @@ module Cnvrg
       return Cnvrg::Result.new(false, "Failed to upload files") unless Cnvrg::CLI.is_response_success(upload_resp, false)
       results = upload_resp['result'].with_indifferent_access
       if results['files'].blank?
+        progressbar.progress += tree.keys.length if progressbar.present?
         return 0
       end
       props = Cnvrg::Helpers.get_s3_props(results)

data/lib/cnvrg/dataset.rb

@@ -9,17 +9,22 @@ module Cnvrg
 
     def initialize(project_home)
       begin
-        config = YAML.load_file(project_home+"/.cnvrg/config.yml")
         @local_path = project_home
+        @working_dir = project_home
+        config = YAML.load_file(project_home+"/.cnvrg/config.yml")
         @title = config[:dataset_name]
         @slug = config[:dataset_slug]
         @owner = config[:owner]
-        @working_dir = project_home
       rescue => e
 
       end
     end
 
+    def change_url(owner: '', slug: '', title: '')
+      config = {dataset_home: title, dataset_slug: slug, owner: owner}
+      File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+    end
+
 
     def self.delete(dataset_slug,owner)
       response = Cnvrg::API.request("users/#{owner}/datasets/#{dataset_slug}/delete", 'DELETE')
@@ -295,10 +300,12 @@ module Cnvrg
       url = Cnvrg::Helpers.remote_url
       "#{url}/#{self.owner}/datasets/#{self.slug}"
     end
-    def generate_chunked_idx(list_files=[], threads: IDXParallelThreads)
+    def generate_chunked_idx(list_files=[], threads: IDXParallelThreads, prefix: '')
       tree = {}
-      Parallel.map(list_files, in_threads: IDXParallelThreads) do |file|
+      Parallel.map(list_files, in_threads: threads) do |file|
+        #check if prefix exists do prefix/path otherwise path
         label = file.gsub(self.local_path + "/", "")
+        label = "#{prefix}/#{label}" if prefix.present?
         if File.directory? file
           tree[label+"/"] = nil
         else
@@ -308,9 +315,19 @@ module Cnvrg
           mime_type = MimeMagic.by_path(file)
           content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
           relative_path = file.gsub(/^#{@local_path + "/"}/, "")
+          relative_path = "#{prefix}/#{relative_path}" if prefix.present?
           tree[label] = {sha1: sha1, file_name: file_name, file_size: file_size, content_type: content_type, absolute_path: file, relative_path: relative_path}
         end
       end
+      if prefix.present? #add the prefix as dirs to the files
+        #lets say the prefix is a/b/c so we want that a/, a/b/, a/b/c/ will be in our files_list
+        dirs = prefix.split('/')
+        curr_path = []
+        dirs.each do |dir|
+          curr_path << dir
+          list_files << curr_path.join('/')
+        end
+      end
       return tree
     end
 

data/lib/cnvrg/files.rb

@@ -12,10 +12,11 @@ module Cnvrg
 
     attr_reader :base_resource
 
-    def initialize(owner, project_slug)
+    def initialize(owner, project_slug, project_home: '')
       @project_slug = project_slug
       @owner = owner
       @base_resource = "users/#{owner}/projects/#{project_slug}/"
+      @project_home = project_home
     end
 
     def upload_file(absolute_path, relative_path, commit_sha1)
@@ -657,9 +658,9 @@ module Cnvrg
       return true
 
     end
-      def start_commit(new_branch,force:false, exp_start_commit:nil, job_slug: nil, job_type: nil)
+      def start_commit(new_branch,force:false, exp_start_commit:nil, job_slug: nil, job_type: nil, start_commit: nil)
         response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug, new_branch: new_branch,force:force,
-                                                                                username: @owner, exp_start_commit:exp_start_commit, job_slug: job_slug, job_type: job_type})
+                                                                                username: @owner, exp_start_commit:exp_start_commit, job_slug: job_slug, job_type: job_type, start_commit: start_commit})
         Cnvrg::CLI.is_response_success(response,false)
         return response
       end
@@ -669,6 +670,60 @@ module Cnvrg
       return response
     end
 
+    def download_file(file_path: '', key: '', iv: '', bucket: '', path: '', client: nil)
+      file_key =  Cnvrg::Helpers.decrypt(key,iv, path)
+      resp = false
+      File.open(@project_home+"/"+file_path, 'w+') do |file|
+        resp = client.get_object({bucket:bucket,
+                                  key:file_key}, target: file)
+      end
+    end
+
+    def delete(file)
+      FileUtils.rm_rf(file)
+    end
+
+    def handle_compare_idx(compared, resolver: {})
+      begin
+        all_files = compared.values.flatten.uniq
+        props = Cnvrg::Helpers.get_s3_props(resolver)
+        files = resolver['keys'].map{|f| [f['name'], f]}.to_h
+        client = props[:client]
+        iv = props[:iv]
+        key = props[:key]
+        bucket = props[:bucket]
+        parallel_options = {
+            :progress => {
+                :title => "Jump Progress",
+                :progress_mark => '=',
+                :format => "%b>>%i| %p%% %t",
+                :starting_at => 0,
+                :total => all_files.size,
+                :autofinish => true
+            },
+            in_processes: Cnvrg::CLI::ParallelProcesses,
+            in_thread: Cnvrg::CLI::ParallelThreads
+        }
+        Parallel.map(all_files, parallel_options) do |file|
+          Cnvrg::CLI.log_message("Trying #{file}")
+          if compared['conflicts'].include? file
+            self.download_file(file_path: "#{file}.conflict", key: key, iv: iv, bucket: bucket, path: files[file]['path'], client: client)
+            next
+          end
+          if compared['updated_on_server'].include? file or compared['added'].include? file
+            self.download_file(file_path: file, key: key, iv: iv, bucket: bucket, path: files[file]['path'], client: client)
+            next
+          end
+          if compared['deleted'].include? file
+            self.delete(file)
+            next
+          end
+          Cnvrg::CLI.log_message("Failed #{file}")
+        end
+      rescue => e
+        raise Exception.new("Cant upload files")
+      end
+    end
       def rollback_commit(commit_sha1)
         response = Cnvrg::API.request("#{base_resource}/commit/rollback", 'POST', {commit_sha1: commit_sha1})
         Cnvrg::CLI.is_response_success(response, false)

data/lib/cnvrg/project.rb

@@ -1,7 +1,7 @@
 require 'fileutils'
 module Cnvrg
   class Project
-    attr_reader :slug, :owner, :title, :local_path, :working_dir, :is_git
+    attr_reader :slug, :owner, :title, :local_path, :working_dir, :is_git, :is_branch
 
     RemoteURL ||= "https://cnvrg.io"
     IDXParallelThreads ||= 15
@@ -15,10 +15,10 @@ module Cnvrg
         @title = config[:project_name]
         @slug = config[:project_slug]
         @owner = config[:owner]
+        @is_branch = config[:is_branch]
         @is_git = config[:git] || false
       rescue => e
       end
-
     end
 
 
@@ -423,14 +423,26 @@ module Cnvrg
         CLI.is_response_success(response,true)
         return response
       end
-    def jump_idx(new_branch, commit=last_local_commit)
-
+    def jump_idx(destination: self.last_local_commit)
       local_idx = self.generate_idx
-      ignore_list = self.send_ignore_list()
-      response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/jump", 'POST', {idx: local_idx, new_branch: new_branch,
-                                                                                                 current_commit: commit,ignore:ignore_list})
+      ignore_list = self.send_ignore_list
+      current_commit = local_idx[:commit]
+      tree = local_idx[:tree]
+      response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/jump_to",
+                                    'POST',
+                                    {tree: tree, ignore: ignore_list,
+                                     dest_commit: destination, current_commit: current_commit})
       CLI.is_response_success(response,false)
-      return response
+      response
+    end
+
+    def set_on_branch(is_latest)
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      @is_branch = !is_latest
+      config[:is_branch] = @is_branch
+
+      File.open(@working_dir+"/.cnvrg/config.yml", 'w') { |f| f.write config.to_yaml }
+
     end
 
       def compare_commit(commit)

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '0.7.0'
+  VERSION = '0.7.1'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.1
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-04 00:00:00.000000000 Z
+date: 2018-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -400,6 +400,7 @@ files:
 - lib/cnvrg/api.rb
 - lib/cnvrg/auth.rb
 - lib/cnvrg/cli.rb
+- lib/cnvrg/colors.rb
 - lib/cnvrg/data.rb
 - lib/cnvrg/datafiles.rb
 - lib/cnvrg/dataset.rb