cnvrg 0.4.0 → 0.4.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d1d1df0dbab52f8c30f1a3e149f04ce2736b7f1c
-  data.tar.gz: 6ee10285e341bffd5436c79bdd3d6036fc355b02
+  metadata.gz: 5feadd09ac750893368f8ce797dab118bb7b2b00
+  data.tar.gz: 2b1ecc2d4ac59d49dae305e86063bdf3cf78666f
 SHA512:
-  metadata.gz: 2416fe3163e8160c1c5c8d9c93e77ccbd4ba388f9bc5ffceb4876c54fb509a5dac7acbb99b81c528717759aa2216d301aaf5eca1fca7a00989538a6428d8d22a
-  data.tar.gz: 100b34a24b210d0ad3de434c79108d02d8aa954b9102d9c33f137674c7906bd23d011929a16d4c46e6d6abadc2ef2b308cd9a390f2beee1f01003aa28f460c8d
+  metadata.gz: b41e7bf46edf0a5fc7f6f111fa59769312412afebe55e497cb32522f87fb060f630767adae7959ebcac01959f6f100a464efc8364cd47078162c2794cd987100
+  data.tar.gz: 9e4aac7444147003fb95d65a44a4490b463e98b1897cfce86636330ca8af1686e629acec25ca6f345c32b70725fe328a4da09d87de2642b102a8578e71b48e74

data/lib/cnvrg/cli.rb

@@ -22,6 +22,7 @@ require 'cnvrg/dataset'
 require 'cnvrg/datafiles'
 require 'cnvrg/data'
 require 'cnvrg/ssh'
+require 'cnvrg/result'
 require 'logstash-logger'
 require 'cnvrg/job'
 require 'docker'
@@ -68,7 +69,6 @@ class Thor
       stop_on_unknown = Thor.stop_on_unknown_option? config[:current_command]
       opts = Thor::Options.new(parse_options, hash_options, stop_on_unknown)
       real_options = []
-
       real_args = [].replace(array_options)
       if local_options.is_a? (Array) and !local_options.empty? and args.empty?
         array_options.each_with_index do |p, i|
@@ -1579,52 +1579,47 @@ module Cnvrg
           response = @project.clone(remote, commit_to_clone)
           Cnvrg::CLI.is_response_success response
           working_dir = project_home
-          docker_image = response["result"]["image"]
-          current_commit = response["result"]["commit"]
           idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
           File.open(working_dir + "/.cnvrg/idx.yml", "w+") {|f| f.write idx.to_yaml}
-          if !docker_image.nil? and !docker_image.empty? and !remote
-            local_images = Docker::Image.all
-            docker_image_local = local_images.map {|x| x.info["RepoTags"]}.flatten.select {|y| y.eql? "#{docker_image}:latest"}.flatten
-            if docker_image_local.size == 0
-              if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
-                image = pull(docker_image)
-                if image
-                  log_message("downloaded image: #{docker_image}", Thor::Shell::Color::BLUE)
-                  @image = Images.new(working_dir, docker_image)
-                else
-                  log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
-                  @project.revert(working_dir)
-                  exit(1)
-                end
-              end
 
-            elsif docker_image_local.size == 1
-              log_message("found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE)
-              @image = Images.new(working_dir, docker_image_local[0])
-            elsif docker_image_local.size > 1
-              log_message("found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE)
-              image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
-              @image = Images.new(working_dir, image_name)
-            end
+          log_message("Downloading files", Thor::Shell::Color::BLUE)
+          files = response["result"]["files"]
 
-          end
+          if !files.nil?
+            begin
+              download_result = @files.download_multiple_files_s3(files, project_home)
 
-          successful_changes = []
-          log_message("Downloading files", Thor::Shell::Color::BLUE)
-          if !response["result"]["tree"].nil?
-            parallel_options = {
-                :progress => {
-                    :title => "Download Progress",
-                    :progress_mark => '=',
-                    :format => "%b>>%i| %p%% %t",
-                    :starting_at => 0,
-                    :total => response["result"]["tree"].size,
-                    :autofinish => true
-                },
-                in_processes: ParallelProcesses,
-            }
+            rescue Interrupt
+              log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
+
+              @files.revoke_clone(project_home)
+              return
+            end
+            Project.verify_cnvrgignore_exist(project_name, remote)
+            if !download_result.is_success?
+              log_message(download_result.e_msg, download_result.msg_color, false)
+              log_message(download_result.e_msg_backtrace, download_result.msg_color, false)
+
+              @files.revoke_clone(project_home)
+            end
+            log_message(download_result.msg, download_result.msg_color)
+          else
+            current_commit = response["result"]["commit"]
+            if !response["result"]["tree"].nil?
+              parallel_options = {
+                  :progress => {
+                      :title => "Download Progress",
+                      :progress_mark => '=',
+                      :format => "%b>>%i| %p%% %t",
+                      :starting_at => 0,
+                      :total => response["result"]["tree"].size,
+                      :autofinish => true
+                  },
+                  in_processes: ParallelProcesses,
+                  in_thread: ParallelThreads
+              }
             begin
+              successful_changes =[]
               is_success = true
               clone_result = Parallel.map((response["result"]["tree"]), parallel_options) do |f|
 
@@ -1635,51 +1630,55 @@ module Cnvrg
                     f
                     successful_changes << relative_path
                   else
-                    is_success = false
+                    is_success =false
                     log_message("Could not create directory: #{f[0]}", Thor::Shell::Color::RED)
                     raise Parallel::Kill
                   end
                 else
                   # blob
 
-                  if @files.download_file_s3(f[0], relative_path, project_home, commit_sha1 = current_commit)
+                  if @files.download_file_s3(f[0], relative_path, project_home, commit_sha1=current_commit)
                     f
                     successful_changes << relative_path
                   else
-                    is_success = false
+                    is_success =false
                     log_message("Could not download file: #{f[0]}", Thor::Shell::Color::RED)
                     raise Parallel::Kill
 
                   end
                 end
               end
+              successful_changes = response["result"]["tree"]
+              if !successful_changes.nil? and is_success
+                Project.verify_cnvrgignore_exist(project_name,remote)
+                log_message("Done.\nDownloaded  #{successful_changes.size}/#{response["result"]["tree"].size} files", Thor::Shell::Color::GREEN)
+              else
+                log_message("Couldn't download some files", Thor::Shell::Color::RED)
+
+              end
             rescue Interrupt
-              is_success = false
               log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
 
-              @files.revoke_download([], response["result"]["tree"])
-              exit(1)
+              @files.revoke_clone(project_home)
+              return
+            end
             end
 
 
           end
-          successful_changes = response["result"]["tree"]
-          if !successful_changes.nil? and is_success
-            Project.verify_cnvrgignore_exist(project_name, remote)
-            log_message("Done.\nDownloaded  #{successful_changes.size}/#{response["result"]["tree"].size} files", Thor::Shell::Color::GREEN)
-          else
-            log_message("Couldn't download some files", Thor::Shell::Color::RED)
-
-          end
 
         else
 
-          log_message("Error: Couldn't create directory: #{project_name}", Thor::Shell::Color::RED)
-          exit(1)
+          log_message("Error: Couldn't clone: #{project_name}", Thor::Shell::Color::RED)
+          @files.revoke_clone(project_home)
+
+          return
         end
       rescue SignalException
         say "\nAborting"
-        exit(1)
+        @files.revoke_clone(project_home)
+
+        return
       end
 
     end
@@ -2153,6 +2152,7 @@ module Cnvrg
     method_option :message, :type => :string, :aliases => ["-m", "--message"], :default => ""
     method_option :deploy, :type => :boolean, :aliases => ["-d", "--deploy"], :default => false
     method_option :return_id, :type => :boolean, :aliases => ["-r", "--return_id"], :default => false
+    method_option :files, :type => :string, :aliases => ["-f", "--files"], :default => nil
 
     def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false)
 
@@ -2169,6 +2169,10 @@ module Cnvrg
         @files = Cnvrg::Files.new(@project.owner, @project.slug)
         ignore = options[:ignore] || ""
         force = options[:force] || false
+        spec_files_to_upload = options["files"]
+        if !spec_files_to_upload.blank?
+          spec_files_to_upload = spec_files_to_upload.split(",")
+        end
 
         if ignore.nil? or ignore.empty?
           ignore = ignore_list
@@ -2186,7 +2190,7 @@ module Cnvrg
         end
         new_branch = options["new_branch"] || false
 
-        result = @project.compare_idx(new_branch, force: force, deploy: options["deploy"],in_exp:in_exp)
+        result = @project.compare_idx(new_branch, force: force, deploy: options["deploy"],in_exp:in_exp, specific_files: spec_files_to_upload)
         commit = result["result"]["commit"]
         if !link
           if (result["result"]["new_version_exist"] and !force) or ((commit != @project.last_local_commit and !@project.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?) and !force)
@@ -2197,19 +2201,6 @@ module Cnvrg
           log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE, (options["verbose"]))
         end
         result = result["result"]["tree"]
-        # if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
-        #   all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
-        #   if all.size == 1
-        #     num = "conflict"
-        #   else
-        #     num =  "conflicts"
-        #   end
-        #   say "Project contains #{all.size} #{num}:", Thor::Shell::Color::RED
-        #   say "#{all.join("\n")}"
-        #   say "Please fix #{num}, and retry", Thor::Shell::Color::RED
-        #   exit(1)
-        #
-        # end
         check = Helpers.checkmark()
         if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
           log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
@@ -2833,7 +2824,6 @@ module Cnvrg
               else
                 # blob
                 log_message("downloading file: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
-
                 if @files.download_file_s3(f, relative_path, project_home)
                   f
                 else
@@ -3001,14 +2991,13 @@ module Cnvrg
     def show
       path = options['path']
       commit = options['commit']
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        project_home = get_project_home
+        @project = Project.new(project_home)
 
-      verify_logged_in(true)
-      log_start(__method__, args, options)
-      project_home = get_project_home
-      @project = Project.new(project_home)
 
 
-      project_dir = is_cnvrg_dir(Dir.pwd)
       @files = Cnvrg::Files.new(@project.owner, @project.slug)
       begin
 
@@ -3084,6 +3073,7 @@ module Cnvrg
     method_option :return_id, :type => :boolean, :aliases => ["-r", "--return_id"], :default => false
     method_option :deploy, :type => :boolean, :aliases => ["-d", "--deploy"], :default => false
     method_option :in_exp, :type => :boolean, :aliases => ["-e", "--in_exp"], :default => false
+    method_option :files, :type => :string, :aliases => ["-f", "--files"], :default => nil
 
     def sync(direct = true)
       verify_logged_in(true) if direct
@@ -3094,7 +3084,7 @@ module Cnvrg
         invoke :download, [true, "", options["in_exp"] ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
       end
       invoke :upload, [false, true,  direct, "",options["in_exp"] ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
-             :ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"]
+             :ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"], :files => options["files"]
 
 
     end
@@ -3105,9 +3095,9 @@ module Cnvrg
     method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
     method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
     method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
-    method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
-    method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
-    method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
+    method_option :gpu_1, :type => :boolean, :aliases => ["--gpu"], :default => false
+    method_option :gpu_2, :type => :boolean, :aliases => ["--gpuxl"], :default => false
+    method_option :gpu_3, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
     method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sync_before"], :default => true
     method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sync_after"], :default => true
     method_option :title, :type => :string, :aliases => ["-t", "--title"], :default => ""
@@ -3153,7 +3143,7 @@ module Cnvrg
       options_hash = Hash[options]
       real_options = []
       options_hash.each do |o|
-        real_options << o if (!o[1].eql? "" and !["small", "medium", "large", "gpu", "gpuxl", "gpuxxl"].include? o[0])
+        real_options << o if (!o[1].eql? "" and !["small", "medium", "large", "gpu_1", "gpu_2", "gpu_3"].include? o[0])
       end
       if local
         invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
@@ -3175,7 +3165,7 @@ module Cnvrg
         end
         real_options.delete(["local", false])
         instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
-                     "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
+                      "gpu_1" => options["gpu_2"], "gpu_3" => options["gpuxxl"]}
         instance_type = get_instance_type(instances)
         if !instance_type.nil? and !instance_type.empty?
           real_options << ["machine_type", instance_type]
@@ -3212,8 +3202,6 @@ module Cnvrg
     method_option :periodic_sync, :type => :string, :aliases => ["-ps", "--periodic_sync"], :default => ""
     def exec(*cmd)
       log = []
-      cpu_average = 0
-      memory_average = 0
       verify_logged_in(true)
       log_start(__method__, args, options)
       working_dir = is_cnvrg_dir
@@ -3340,6 +3328,24 @@ module Cnvrg
                   end
                 end
               # end
+              stats_thread = Thread.new do
+                loop do
+                  begin
+                    stats = remote ? usage_metrics_in_docker(docker_id) : Helpers.linux? ? {memory: memory_usage, cpu: cpu_usage} : {}
+                    if is_on_gpu
+                      gu = gpu_util
+                      stats['gpu'] = gu[0]
+                      stats['gpu_util'] = gu[1]
+                    end
+                    @exp.send_machine_stats [stats]
+                  rescue => e
+                    log_error(e)
+                    log_message("Failed to upload ongoing stats, continuing with experiment", Thor::Shell::Color::YELLOW)
+                  ensure
+                    sleep(30)
+                  end
+                end
+              end
 
 
               log_thread = Thread.new do
@@ -3349,30 +3355,7 @@ module Cnvrg
                   begin
                     if log_count != 0
                       if time_to_upload <= Time.now - start_loop
-                        if remote
-                          stats = usage_metrics_in_docker(docker_id)
-                          cpu = stats[0]
-                          memory = stats[1]
-                          if is_on_gpu
-                            gpu_stats = gpu_util
-                            gpu_utilization = gpu_stats[0]
-                            gpu_memory_util = gpu_stats[1]
-                          end
-                        else
-                          if Helpers.linux?
-                            memory = memory_usage()
-                            cpu = cpu_usage()
-                          end
-
-                        end
-                        log.each do |l|
-                          if remote and is_on_gpu
-                            l.merge!(cpu: cpu, memory: memory, gpu_util: gpu_utilization, gpu_memory_util: gpu_memory_util)
-                          else
-                            l.merge!(cpu: cpu, memory: memory)
-                          end
-                        end
-                        @exp.upload_temp_log(log, cpu_average, memory_average)
+                        @exp.upload_temp_log(log)
                         log = []
                         start_loop = Time.now
                       end
@@ -3416,12 +3399,9 @@ module Cnvrg
 
                 rescue Errno::EIO => e
                   log_error(e)
-
-                    # break
                 rescue Errno::ENOENT => e
 
                   exp_success = false
-
                   log_message("command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED)
                   log_error(e)
                 rescue Open4::ChildExited
@@ -3429,9 +3409,7 @@ module Cnvrg
                   log_message("The process exited!", Thor::Shell::Color::RED)
                 rescue => e
                   sleep(20) # end cycle
-
                   res = @exp.end(log, 1, start_commit, cpu_average, memory_average)
-
                   log_message("Error occurred,aborting", Thor::Shell::Color::RED)
                   log_error(e)
                   exit(0)
@@ -3488,11 +3466,12 @@ module Cnvrg
                 }
                 log << cur_log
                 sleep(20) # end cycle
-
                 res = @exp.end(log, "-1", end_commit, cpu_average, memory_average)
 
               end
               log_error(e)
+              Thread.kill(log_thread)
+              Thread.kill(stats_thread)
 
               exit(1)
             end
@@ -3664,36 +3643,6 @@ module Cnvrg
         res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
                               periodic_sync, sync_before_terminate, max_time, ds_sync_options)
         if Cnvrg::CLI.is_response_success(res)
-          # if res["result"]["machine"] == -1
-          #   say "There are no available machines", Thor::Shell::Color::BLUE
-          #   create = yes? "create new machine?", Thor::Shell::Color::YELLOW
-          #   if create
-          #     res = Cnvrg::API.request("users/#{image.owner}/machines/list", 'GET')
-          #     if Cnvrg::CLI.is_response_success(res)
-          #       instance_type = machine_options(res["result"]["aws_options"])
-          #       if @image.new_machine(instance_type)
-          #         say "Running remote experiment", Thor::Shell::Color::BLUE
-          #
-          #         # res = image.exec_remote(exec_args, exec_options, project.last_local_commit)
-          #         # if Cnvrg::CLI.is_response_success(res)
-          #
-          #           check = Helpers.checkmark()
-          #           say "#{check} Finished successfuly", Thor::Shell::Color::GREEN
-          #           exit(0)
-          #         # end
-          #       end
-          #     else
-          #       say "No machines are avilable", Thor::Shell::Color::RED
-          #       exit(0)
-          #     end
-          #
-          #
-          #   else
-          #     say "Can't execute command on remote machine with local image", Thor::Shell::Color::RED
-          #     exit(1)
-          #
-          #   end
-          # else
           check = Helpers.checkmark()
           str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
 
@@ -3723,18 +3672,17 @@ module Cnvrg
       end
     end
     desc 'deploy', 'Deploys model to production'
-    method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
-    method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
-    method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
+    method_option :small, :type => :boolean, :aliases => ["-s", "--small"], :default => false
+    method_option :medium, :type => :boolean, :aliases => ["-m", "--medium"], :default => false
+    method_option :large, :type => :boolean, :aliases => ["-l", "--large"], :default => false
     method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
     method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
     method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
-    method_option :schedule, :type => :string, :aliases => ["--schedule", "-s"], :default => ""
-
+    method_option :schedule, :type => :string, :aliases => ["--schedule"], :default => ""
     method_option :commit, :type => :string, :aliases => ["--commit", "-c"], :default => ""
     method_option :workers, :type => :string, :aliases => ["--workers", "-w"], :default => ""
     method_option :file_as_input, :type => :boolean, :aliases => ["--input", "-i"], :default => false
-
+    method_option :title, :type => :string, :aliases => ["--title", "-t"], :default => ""
     def deploy(file_to_run, function)
       verify_logged_in(true)
       log_start(__method__, args, options)
@@ -3744,16 +3692,14 @@ module Cnvrg
                      "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
         instance_type = get_instance_type(instances)
 
-        schedule = options["schedule"] || ""
-
+          schedule = options["schedule"] || ""
+          title = options['title']
 
-        if !schedule.nil? and !schedule.empty?
-
-          local_timestamp = get_schedule_date
-
-        end
-        project = Project.new(working_dir)
-        commit_to_run = options["commit"] || nil
+          if !schedule.nil? and !schedule.empty?
+            local_timestamp = get_schedule_date
+          end
+          project = Project.new(working_dir)
+          commit_to_run = options["commit"] || nil
 
         workers = options["workers"] || nil
         begin
@@ -3767,14 +3713,8 @@ module Cnvrg
 
         image = is_project_with_docker(working_dir)
         if !image or !image.is_docker
-          # say "Couldn't find image related to project", Thor::Shell::Color::RED
-          # default = yes? "use cnvrg default image?", Thor::Shell::Color::YELLOW
-          # if default
           image = Images.new(working_dir, "cnvrg")
           image_slug = image.image_slug
-          # else
-          #   exit(0)
-          # end
         else
           image_slug = image.image_slug
         end
@@ -3782,42 +3722,12 @@ module Cnvrg
 
         invoke :sync, [false], []
 
-        res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp, num_workers, file_as_input)
+          res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp, num_workers, file_as_input, title)
 
         if Cnvrg::CLI.is_response_success(res)
 
-          # if res["result"]["machine"] == -1
-          #   say "There are no available machines", Thor::Shell::Color::BLUE
-          #   create = yes? "create new machine?", Thor::Shell::Color::YELLOW
-          #   if create
-          #     res = Cnvrg::API.request("users/#{image.owner}/machines/list", 'GET')
-          #     if Cnvrg::CLI.is_response_success(res)
-          #       instance_type = machine_options(res["result"]["aws_options"])
-          #       if @image.new_machine(instance_type)
-          #         say "Running remote experiment", Thor::Shell::Color::BLUE
-          #
-          #         # res = image.exec_remote(exec_args, exec_options, project.last_local_commit)
-          #         # if Cnvrg::CLI.is_response_success(res)
-          #
-          #           check = Helpers.checkmark()
-          #           say "#{check} Finished successfuly", Thor::Shell::Color::GREEN
-          #           exit(0)
-          #         # end
-          #       end
-          #     else
-          #       say "No machines are avilable", Thor::Shell::Color::RED
-          #       exit(0)
-          #     end
-          #
-          #
-          #   else
-          #     say "Can't execute command on remote machine with local image", Thor::Shell::Color::RED
-          #     exit(1)
-          #
-          #   end
-          # else
-          check = Helpers.checkmark()
-          log_message("#{check} Deployment process is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/deploys/show/#{res["result"]["deploy_slug"]}", Thor::Shell::Color::GREEN)
+            check = Helpers.checkmark()
+            log_message("#{check} Deployment process is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/endpoints/show/#{res["result"]["deploy_slug"]}", Thor::Shell::Color::GREEN)
 
           exit(0)
           # end
@@ -6057,31 +5967,6 @@ module Cnvrg
         history = container.exec(command, tty: true)[0]
         diff = [pip, dpkg, history]
         return diff
-        # File.open(dir+"/.cnvrg/pip_#{date}.txt", "w+") { |f| f.write pip }
-        # File.open(dir+"/.cnvrg/dpkg_#{date}.txt", "w+") { |f| f.write dpkg }
-        # File.open(dir+"/.cnvrg/history_#{date}.txt", "w+") { |f| f.write history }
-        #
-        #   pip_new = pip[0].split("\r\n")
-        #   pip_base_file =JSON.parse(File.open( dir+"/.cnvrg/pip_base.txt","rb").read)
-        #   pip_base = pip_base_file[0].split("\r\n")
-        #
-        #   pip_diff = pip_new - pip_base
-        #   if pip_diff.empty?
-        #     puts "pip is same"
-        #   else
-        #     puts pip_diff
-        #   end
-        #   dpkg_new = dpkg[0].split("\r\n")
-        #   dpkg_base_file =JSON.parse(File.open( dir+"/.cnvrg/dpkg_base.txt","rb").read)
-        #   dpkg_base = dpkg_base_file[0].split("\r\n")
-        #
-        #   dpkg_diff = dpkg_new - dpkg_base
-        #   if dpkg_diff.empty?
-        #     puts "dpkg is same"
-        #   else
-        #     puts dpkg_diff
-        #   end
-        #   puts history
       end
 
       def is_port_taken(ip = Cnvrg::CLI::IP, port = Cnvrg::CLI::PORT, seconds = 1)
@@ -6098,7 +5983,7 @@ module Cnvrg
       end
 
       def image_activity(image)
-        res = image.handle_image_activity()
+        res = image.handle_image_activity
         if res == -1
           #image is not known
           say "Images #{image.image_name} is not known", Thor::Shell::Color::YELLOW
@@ -6132,6 +6017,29 @@ module Cnvrg
 
       end
 
+
+
+        def format_schedule(schedule)
+          if schedule.start_with? 'in'
+            time = schedule.split(" ")
+
+            local = Time.now.localtime
+            if time[2].downcase().start_with? "min"
+              new = local + (time[1].to_i * 60)
+            elsif time[2].downcase().start_with? "hours"
+              new = local + (time[1].to_i * 3600)
+            elsif time[2].downcase().start_with? "days"
+              new = local + (time[1].to_i * 3600 * 24)
+            else
+              log_message("Could not undersatnd when to schedule experiment", Thor::Shell::Color::RED)
+              exit(1)
+            end
+            new_time = new.to_s
+            new_time = new_time[0, new_time.size-6] #remove timezone
+            schedule = "at #{new_time}"
+          end
+          schedule
+        end
       def update_deleted(deleted)
         final = []
         deleted.each do |d|
@@ -6139,7 +6047,7 @@ module Cnvrg
           to_add = true
           value = all_subs[0]
           all_subs.each_with_index do |a, i|
-            if final.include? value + "/"
+            if final.include? value+"/"
               to_add = false
               break
             end
@@ -6147,7 +6055,6 @@ module Cnvrg
           end
           final << d if to_add
 
-
         end
 
 
@@ -6258,7 +6165,7 @@ module Cnvrg
       end
 
       def usage_metrics_in_docker(docker_id)
-        res = [0.0, 0.0]
+        res = {cpu: 0.0, memory: 0.0}
         begin
           if docker_id.nil?
             docker_id = `cat /etc/hostname`
@@ -6268,7 +6175,7 @@ module Cnvrg
             conv = stats.split(",")
             cpu = conv[0].gsub!("%", "").to_f
             memory = conv[1].gsub!("%", "").to_f
-            res = [cpu, memory]
+            res = {cpu: cpu, memory: memory}
             return res
           end
         rescue

data/lib/cnvrg/experiment.rb

@@ -103,12 +103,17 @@ module Cnvrg
       return response
     end
 
-    def upload_temp_log(temp_log, cpu_average, memory_average)
+    def upload_temp_log(temp_log)
       response = Cnvrg::API.request(@base_resource + "experiment/upload_temp_log", 'POST', {output: temp_log,
                                                                                             exp_slug: @slug})
       Cnvrg::CLI.is_response_success(response,false)
     end
 
+    def send_machine_stats(stats)
+      response = Cnvrg::API.request(@base_resource + "experiment/upload_stats", "POST", {exp_slug: @slug, stats: stats.map{|s| s.merge!({time: Time.now})}})
+      Cnvrg::CLI.is_response_success(response,false)
+    end
+
     def end(output, exit_status, end_commit, cpu_average, memory_average)
       #if remote try to remove
       response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', {output: output, exp_slug: @slug,

data/lib/cnvrg/files.rb

@@ -3,6 +3,7 @@ require 'aws-sdk'
 require 'URLcrypt'
 require 'tempfile'
 require 'net/http'
+require 'cnvrg/result'
 module Cnvrg
   class Files
 
@@ -373,6 +374,7 @@ module Cnvrg
         return Cnvrg::CLI.is_response_success(response, false)
       end
 
+
       def download_file_s3(absolute_path, relative_path, project_home, commit_sha1=nil, conflict=false)
         begin
           res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path,
@@ -402,6 +404,7 @@ module Cnvrg
               end
             end
             if !success
+              puts "error in sts"
               return false
             end
 
@@ -445,11 +448,127 @@ module Cnvrg
           end
 
         rescue => e
-            puts e.message
+          puts "error in aws"
+
+          puts e.message
             return false
 
         end
       end
+    def download_multiple_files_s3(files, project_home)
+      begin
+
+          sts_path = files["path_sts"]
+          retries = 0
+          success= false
+          while !success and retries < 20
+            begin
+              if !Helpers.is_verify_ssl
+                body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
+              else
+                body = open(sts_path).read
+              end
+              success = true
+            rescue => e
+              retries +=1
+              sleep(5)
+
+            end
+          end
+          if !success
+            return Cnvrg::Result.new(false,"couldn't download some files", "error in sts", "" )
+          end
+          split = body.split("\n")
+          key = split[0]
+          iv = split[1]
+
+          access =  Cnvrg::Helpers.decrypt(key, iv, files["sts_a"])
+
+          secret =  Cnvrg::Helpers.decrypt(key,iv, files["sts_s"])
+
+          session =  Cnvrg::Helpers.decrypt(key,iv, files["sts_st"])
+          region =  Cnvrg::Helpers.decrypt(key,iv, files["region"])
+
+          bucket =  Cnvrg::Helpers.decrypt(key,iv, files["bucket"])
+          is_s3 = files["is_s3"]
+          if is_s3 or is_s3.nil?
+            client = Aws::S3::Client.new(
+                :access_key_id =>access,
+                :secret_access_key => secret,
+                :session_token => session,
+                :region => region,
+                :http_open_timeout => 60, :retry_limit => 20)
+          else
+            endpoint = Cnvrg::Helpers.decrypt(key,iv, files["endpoint_url"])
+            client = Aws::S3::Client.new(
+                :access_key_id =>access,
+                :secret_access_key => secret,
+                :region => region,
+                :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
+                :http_open_timeout => 60, :retry_limit => 20)
+          end
+          download_succ_count = 0
+          parallel_options = {
+              :progress => {
+                  :title => "Download Progress",
+                  :progress_mark => '=',
+                  :format => "%b>>%i| %p%% %t",
+                  :starting_at => 0,
+                  :total => files["keys"].size,
+                  :autofinish => true
+              },
+              in_threads: Cnvrg::Helpers.parallel_threads,
+              isolation: true
+          }
+          Parallel.map(files["keys"], parallel_options) do |f|
+
+            file_path = f["name"]
+            if file_path.end_with? "/"
+              # dir
+              if download_dir(file_path, file_path, project_home)
+                download_succ_count += 1
+              else
+                return Cnvrg::Result.new(false,"Could not create directory: #{file_path}")
+                raise Parallel::Kill
+              end
+            else
+              # blob
+              begin
+              file_key =  Cnvrg::Helpers.decrypt(key,iv, f["path"])
+              resp = false
+              File.open(project_home+"/"+file_path, 'w+') do |file|
+                resp = client.get_object({bucket:bucket,
+                                          key:file_key}, target: file)
+              end
+              if resp
+                download_succ_count +=1
+              else
+                return Cnvrg::Result(false,"Could not create file: #{file_path}")
+              end
+
+
+
+              rescue => e
+                return Cnvrg::Result.new(false,"Could not create file: #{file_path}", e.message, e.backtrace)
+                raise Parallel::Kill
+              end
+
+
+
+              end
+          end
+        if download_succ_count == files["keys"].size
+          return Cnvrg::Result.new(true,"Done.\nDownloaded #{download_succ_count} files")
+        end
+      rescue => e
+
+        return Cnvrg::Result.new(false,"Could not download some files", e.message, e.backtrace)
+      end
+
+
+
+
+    end
 
       def download_file(absolute_path, relative_path, project_home, conflict=false)
         res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
@@ -560,6 +679,13 @@ module Cnvrg
         return true
 
       end
+    def revoke_clone(project_home)
+      begin
+        FileUtils.rm_rf(project_home)
+      rescue
+      end
+
+    end
     def delete_commit_files_local(deleted)
       begin
         FileUtils.rm_rf(deleted) unless (deleted.nil? or deleted.empty?)

data/lib/cnvrg/helpers.rb

@@ -2,7 +2,10 @@ module Cnvrg
   module Helpers
 
     extend self
-
+    ParallelThreads ||= 10
+    def parallel_threads()
+      return ParallelThreads
+    end
     def checkmark
       checkmark = "\u2713"
       return checkmark.encode('utf-8')

data/lib/cnvrg/project.rb

@@ -356,16 +356,18 @@ module Cnvrg
         return response
       end
 
-      def compare_idx(new_branch, commit:last_local_commit,force:false, deploy: false, in_exp:false)
+      def compare_idx(new_branch, commit:last_local_commit,force:false, deploy: false, in_exp:false, specific_files: [])
         if commit.nil?
           local_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
         else
           local_idx = self.generate_idx(deploy: deploy)
         end
         ignore_list = self.send_ignore_list()
-        if force
+        if force or !specific_files.blank?
           added = []
-          if local_idx[:tree]
+          if !specific_files.blank?
+            added = specific_files
+          elsif local_idx[:tree]
             added << local_idx[:tree].keys
             added.flatten!
           end
@@ -413,14 +415,15 @@ module Cnvrg
         return true
       end
 
-      def deploy(file_to_run, function, input_params, commit_to_run, instance_type, image_slug, scheduling_query, local_timestamp,workers, file_input)
+      def deploy(file_to_run, function, input_params, commit_to_run, instance_type, image_slug, scheduling_query, local_timestamp,workers, file_input, title)
         response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/deploy", 'POST', {file_to_run: file_to_run, function: function,
                                                                                            image_slug: image_slug, input_params: input_params,
                                                                                            commit_sha1: commit_to_run,
                                                                                            instance_type: instance_type,
                                                                                            scheduling_query: scheduling_query,
                                                                                            local_timestamp: local_timestamp,
-                                                                                            workers:workers,file_input:file_input})
+                                                                                            workers:workers,file_input:file_input,
+                                                                                            title: title})
         return response
       end
     def list_commits

data/lib/cnvrg/result.rb

@@ -0,0 +1,29 @@
+require 'fileutils'
+module Cnvrg
+  class Result
+    attr_reader :msg, :msg_color, :e_msg, :e_msg_backtrace
+
+    def initialize(success, msg, e_msg = "", e_backtrace = "")
+      begin
+       @success = success
+        @msg = msg
+       @e_msg = e_msg
+       @e_msg_backtrace = e_backtrace
+        if !@success
+          @msg_color = Thor::Shell::Color::RED
+        else
+          @msg_color = Thor::Shell::Color::GREEN
+
+        end
+      rescue => e
+      end
+
+    end
+    def is_success?
+      return @success
+    end
+
+  end
+
+
+  end

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '0.4.0'
+  VERSION = '0.4.1.4'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1.4
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-02 00:00:00.000000000 Z
+date: 2018-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -422,6 +422,7 @@ files:
 - lib/cnvrg/helpers.rb
 - lib/cnvrg/job.rb
 - lib/cnvrg/project.rb
+- lib/cnvrg/result.rb
 - lib/cnvrg/runner.rb
 - lib/cnvrg/ssh.rb
 - lib/cnvrg/version.rb