cnvrg 1.11.31 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a31d199ccd8981e455384ced40cc63a0a9a687982338cbc85e3a9413c2968a2c
-  data.tar.gz: 10d4da464f6ef7b507d563e2976cf17f5d6cd8bcfeab742c83ec5bfa1ab575e2
+  metadata.gz: ab82882b2bb6c9093751cd560eaa4ccb7540fad9b9a34a81245721538dd37a5b
+  data.tar.gz: e35299be744d985a37794288a269ba2bb55cf64d3fcd702c6c6147bd4f5d740d
 SHA512:
-  metadata.gz: 2495cc0d77ada40ae2c6da54ada72710f28facba737b0a37c0dd2cb904d37d32dd23d2cba476fb3854ec47e19eaa11fb34913c2038d6ea059378ad0d460e905a
-  data.tar.gz: dbe50928a5d25259f78fc1dcdab474545b71308b7e1d798678fb2cd8786f0b0b2eda78062bcef6bcfdb16c9481bc47349c6f571de9cfe89f547c57f583bea403
+  metadata.gz: 52b51bb4942583e9ac3eceab24891d252ef7e31d7f1a2a87d5c8f4586f914f33f2d9cc7addc09ab5aa3652a9ba939ef5a37bdb9fc82d18e4e0e0a61126265d17
+  data.tar.gz: df735e631778b5d36d296c33903719a2ba7832ce2a9e218eea033ad246f8c4ce044a7b7d5562f68d901a1ee32d62a88559f886da2a0341d8bf7c51b50c25660c

data/Readme.md

@@ -18,4 +18,9 @@
 ## Version v1.11.30
 2021-04-06
 ## Version v1.11.31
-2021-04-22
+2021-04-22
+## Version v1.11.32
+2021-05-05
+* DEV-8868 - Bug: SDK - e.sync() in git project only creates empty "output" folder in commit
+## Version v2.0.1
+2021-06-13

data/cnvrg.gemspec

@@ -6,7 +6,7 @@ require 'cnvrg/version'
 Gem::Specification.new do |spec|
     spec.name          = 'cnvrg'
     spec.version       = Cnvrg::VERSION
-    spec.authors       = ['Yochay Ettun', 'Leah Kolben']
+    spec.authors       = ['Yochay Ettun', 'Leah Kolben', 'Omer Shacham']
     spec.email         = ['info@cnvrg.io']
     spec.summary       = %q{A CLI tool for interacting with cnvrg.io.}
     spec.description   = %q{A CLI tool for interacting with cnvrg.io.}
@@ -39,6 +39,7 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'google-cloud-storage', '~> 1.21.1'
     spec.add_runtime_dependency 'sucker_punch', '~> 2.0'
     spec.add_runtime_dependency 'urlcrypt', '~> 0.1.1'
+    spec.add_runtime_dependency 'filewatch', '~> 0.9.0'
     spec.add_runtime_dependency 'parallel', '~> 1.12.0'
     spec.add_runtime_dependency 'azure-storage-blob', '~> 1.1.0'
     spec.add_runtime_dependency 'logstash-logger', '~> 0.22.1'

data/lib/cnvrg/api.rb

@@ -72,11 +72,11 @@ module Cnvrg
                     if response.to_hash[:status].to_i != 200
                       Cnvrg::Logger.log_info("Got back bad status #{response.to_hash[:status]}")
                     end
-                    if [503, 502, 429].include?(response.to_hash[:status].to_i)
+                    if [503, 502, 429, 401].include?(response.to_hash[:status].to_i)
                       Cnvrg::Logger.log_info("Got back status #{response.to_hash[:status]}, will retry in #{5 * retries} seconds")
                       success = false
                       sleep(5 * retries)
-                      retries +=1
+                      retries += 1
                       next
                     end
                   rescue => e
@@ -112,11 +112,11 @@ module Cnvrg
                         if response.to_hash[:status].to_i != 200
                           Cnvrg::Logger.log_info("Got back bad status #{response.to_hash[:status]}")
                         end
-                        if [503, 502, 429].include?(response.to_hash[:status].to_i)
+                        if [503, 502, 429, 401].include?(response.to_hash[:status].to_i)
                           Cnvrg::Logger.log_info("Got back status #{response.to_hash[:status]}, will retry in #{5 * retries} seconds")
                           success = false
                           sleep(5 * retries)
-                          retries +=1
+                          retries += 1
                           next
                         end
                     rescue => e

data/lib/cnvrg/cli.rb

@@ -173,7 +173,7 @@ module Cnvrg
     desc "data [COMMAND]", "Upload and manage datasets", :hide => false
     subcommand "data", Data
 
-    desc "job", "manage running jobs", :hide => false
+    desc "job", "manage running jobs", :hide => true
     subcommand "job", JobCli
 
     desc "ssh", "ssh into running jobs", :hide => false
@@ -415,7 +415,7 @@ module Cnvrg
       end
     end
 
-    desc 'set_compression_path', 'Set compression path'
+    desc 'set_compression_path', 'Set compression path', :hide => true
     method_option :reset, :type => :boolean, :aliases => ["-r", "--reset"], :default => false
 
     def set_compression_path(*compression_path)
@@ -2311,6 +2311,7 @@ module Cnvrg
     method_option :suppress_exceptions, :type => :boolean, :aliases => ["--suppress-exceptions"], :default => true
     method_option :debug_mode, :type => :boolean, :aliases => ["--debug-mode"], :default => false
     method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 1000
+    method_option :local, :type => :boolean, :aliases => ["--local"], :default => true
 
     def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false, output_dir = "output", job_type = nil, job_slug = nil, suppress_exceptions = true,chunk_size=1000)
       begin
@@ -2328,6 +2329,8 @@ module Cnvrg
           exp_obj = nil
         end
 
+        local = options["local"]
+
         commit_msg = options["message"]
         if commit_msg.nil? or commit_msg.empty?
           commit_msg = ""
@@ -2349,7 +2352,7 @@ module Cnvrg
             if git_output_dir.ends_with? "/"
               git_output_dir = git_output_dir[0..-2]
             end
-            list = @project.generate_output_dir(git_output_dir)
+            list = @project.generate_output_dir(git_output_dir, local: local)
           end
           list += @project.generate_git_diff if options["git_diff"]
           spec_files_to_upload = list
@@ -2668,7 +2671,7 @@ module Cnvrg
       end
     end
 
-    desc 'commit before termination', 'Commit job code before termination'
+    desc 'commit before termination', 'Commit job code before termination', :hide => true
     def commit_before_termination()
       job_type = ENV['CNVRG_JOB_TYPE']
       job_id =  ENV['CNVRG_JOB_ID']
@@ -2678,7 +2681,7 @@ module Cnvrg
       log_error(e)
     end
 
-    desc 'update_job_commit', 'Update job with its last commit'
+    desc 'update_job_commit', 'Update job with its last commit' , :hide => true
     def update_job_commit()
       job_type = ENV['CNVRG_JOB_TYPE']
       job_id =  ENV['CNVRG_JOB_ID']
@@ -2868,7 +2871,7 @@ module Cnvrg
 
 
 
-    desc 'jump', 'Jump to specific commit'
+    desc 'jump COMMIT_ID', 'Jump to specific commit'
     def jump(commit_sha1)
       begin
         verify_logged_in()
@@ -3008,6 +3011,7 @@ module Cnvrg
     method_option :suppress_exceptions, :type => :boolean, :aliases => ["--suppress-exceptions"], :default => true
     method_option :debug_mode, :type => :boolean, :aliases => ["--debug-mode"], :default => false
     method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 1000
+    method_option :local, :type => :boolean, :aliases => ["--local"], :default => false
 
     def sync(direct = true)
       verify_logged_in(true) if direct
@@ -3030,10 +3034,10 @@ module Cnvrg
       if run_download or options['debug_mode']
         invoke :download, [true, "", in_exp ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
       end
-      invoke :upload, [false, true,  direct, "",in_exp,options[:force], output_dir, job_type, job_slug ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
+      invoke :upload, [false, true, direct, "", in_exp, options[:force], output_dir, job_type, job_slug], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
              :ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"],
              :files => options["files"], :output_dir => output_dir, :job_slug => job_slug, :job_type => job_type, :suppress_exceptions => options["suppress_exceptions"],
-             :debug_mode => options['debug_mode'], :git_diff => options["git_diff"], :chunk_size => options["chunk_size"]
+             :debug_mode => options['debug_mode'], :git_diff => options["git_diff"], :chunk_size => options["chunk_size"], :local => options["local"]
 
     end
 
@@ -3316,62 +3320,47 @@ module Cnvrg
                 end
               end
               start_time = Time.now
-              shell_type = options["use_bash"] ? "bash -l" : "sh"
               if @exp.get_cmd.present?
                 cmd = @exp.get_cmd
-                if options["docker_id"].present? # Escape for docker exec
-                  cmd = cmd.gsub("\"", "\\\"")
-                end
               end
-              if options["docker_id"].present?
-                  cmd = "docker exec -it #{options["docker_id"]} #{shell_type} -c \"#{cmd}\""
+              command_slug = (0...18).map { (65 + rand(26)).chr }.join
+              result_file = "/conf/result-#{command_slug}"
+              data = {cmd: cmd, async: true, format: true, file_name: result_file, use_script: true, use_bash: options["use_bash"]}
+
+              conn = Cnvrg::Helpers::Executer.get_main_conn
+              response = conn.post('command', data.to_json)
+              if response.to_hash[:status].to_i != 200
+                exit_status = 129
+                raise StandardError.new("Cant send command to slave")
               end
-              PTY.spawn(@exp.as_env, cmd) do |stdout, stdin, pid, stderr|
+              t = FileWatch::Tail.new
+              filename = result_file
+              lines = []
+              t.tail(filename)
+              t.subscribe do |path, line|
                 begin
-                  stdout.each do |line|
-                    cur_time = Time.now
-                    real_time = Time.now - real
-                    cur_log = {time: cur_time,
-                               message: line,
-                               type: "stdout",
-                               real: real_time
-                    }
-                    if print_log
-                      puts({log: line, timestamp: Time.now, exp_logs: true}.to_json)
-                    end
+                  cur_log = JSON.parse(line)
+                  if cur_log["type"] == "endMessage"
+                    exit_status = cur_log["real"].to_i
+                    break
+                  else
+                    puts(cur_log.to_json)
+                    STDOUT.flush
+                    cur_log["time"] = Time.parse(cur_log["timestamp"])
+                    cur_log["message"] = cur_log["message"].to_s + "\r\n"
                     log << cur_log
-                    if log.size >= 10
-                      @exp.upload_temp_log(log) unless log.empty?
-                      log = []
-                      elsif (start_time + 15.seconds) <= Time.now
-                      @exp.upload_temp_log(log) unless log.empty?
-                      log = []
-                      start_time = Time.now
-                    end
                   end
-                  if stderr
-                    stderr.each do |err|
-                      log << {time: Time.now, message: err, type: "stderr"}
-                    end
-                  end
-                rescue Errno::EIO => e
-                  log_error(e)
-                  if !log.empty?
-                    temp_log = log
-                    @exp.upload_temp_log(temp_log) unless temp_log.empty?
-                    log -= temp_log
+                  if log.size >= 10
+                    @exp.upload_temp_log(log)
+                    log = []
+                  elsif (start_time + 15.seconds) <= Time.now
+                    @exp.upload_temp_log(log) unless log.empty?
+                    log = []
+                    start_time = Time.now
                   end
-                rescue Errno::ENOENT => e
-                  exp_success = false
-                  log_message("command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED)
-                  log_error(e)
                 rescue => e
-                  res = @exp.end(log, 1, start_commit, 0, 0)
-                  log_message("Error occurred,aborting", Thor::Shell::Color::RED)
                   log_error(e)
-                  exit(0)
                 end
-                ::Process.wait pid
               end
               end_time = Time.now
               process_running = false
@@ -3379,14 +3368,13 @@ module Cnvrg
               if !log.empty?
 
                 temp_log = log
-                  @exp.upload_temp_log(temp_log) unless temp_log.empty?
+                @exp.upload_temp_log(temp_log)
                 log -= temp_log
               end
 
               cpu_average = cpu_total.inject(0) {|sum, el| sum + el}.to_f / cpu_total.size
               memory_average = memory_total.inject(0) {|sum, el| sum + el}.to_f / memory_total.size
-              exit_status = $?.exitstatus
-              if $?.exitstatus != 0
+              if exit_status != 0
                 exp_success = false
               end
 
@@ -3430,7 +3418,6 @@ module Cnvrg
               if @exp
                 # log_thread.join
                 Thread.kill(stats_thread) if docker_stats
-                exit_status = $?.exitstatus
                 if exit_status.blank?
                   exit_status = "-1"
                 end
@@ -3443,8 +3430,6 @@ module Cnvrg
 
               exit(1)
             end
-
-
           end
 
         end
@@ -3689,7 +3674,7 @@ module Cnvrg
       end
     end
 
-    desc 'deploy', 'Deploys model to production'
+    desc 'deploy', 'Deploys model to production', :hide => true
     method_option :small, :type => :boolean, :aliases => ["-s", "--small"], :default => false
     method_option :medium, :type => :boolean, :aliases => ["-m", "--medium"], :default => false
     method_option :large, :type => :boolean, :aliases => ["-l", "--large"], :default => false
@@ -3778,7 +3763,7 @@ module Cnvrg
     method_option :dataset_only_tree, :type => :boolean, :aliases => [ "--dataset_only_tree"], :default => false
     method_option :data_query, :type => :string, :aliases => ["-q", "--query"], :default => ""
 
-    desc 'notebook', 'Starts a notebook session remotely or locally'
+    desc 'notebook', 'Starts a notebook session remotely or locally', :hide => true
 
     def notebook
       verify_logged_in(true)
@@ -3905,7 +3890,7 @@ module Cnvrg
       end
     end
 
-    desc 'remote_notebook', 'Run notebook server on remote server'
+    desc 'remote_notebook', 'Run notebook server on remote server', :hide => true
     method_option :machine_type, :type => :string, :default => ""
     method_option :notebook_type, :type => :string, :aliases => ["-n", "--notebook_type"], :default => ""
     method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
@@ -4264,7 +4249,7 @@ module Cnvrg
 
     end
 
-    desc 'notebook_stop', 'Starts a new notebook environment'
+    desc 'notebook_stop', 'Stop notebook', :hide => true
     method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
     method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false, :desc => "run on remote machine"
     method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
@@ -4651,10 +4636,16 @@ module Cnvrg
     method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :desc => "collect gpu metrics", :default => false
     method_option :gaudi, :type => :boolean, :aliases => ["--gaudi"], :desc => "collect gaudi metrics", :default => false
     method_option :wait, :type => :numeric, :aliases => ["--wait"], :desc => "to to wait between querying", :default => 30
+    method_option :prom_user, :type => :string, :aliases => ["--prom_user"], :desc => "prometheus username", :default => nil
+    method_option :prom_password, :type => :string, :aliases => ["--prom_password"], :desc => "prometheus password", :default => nil
+    method_option :name, :type => :string, :aliases => ["--name"], :desc => "pod name - used for master-workers jobs", :default => nil
 
     def collect_metrics
       @exp = Experiment.new(ENV['CNVRG_OWNER'], ENV['CNVRG_PROJECT'], job_id: ENV['CNVRG_JOB_ID'])
       prometheus_url = options[:prometheus_url].ends_with?("/") ? options[:prometheus_url] : "#{options[:prometheus_url]}/"
+      prom_user = options[:prom_user]
+      prom_password = options[:prom_password]
+      name = options[:name]
 
       translate_result = Cnvrg::API_V2.request(
         "#{ENV['CNVRG_OWNER']}/resources/translate_metrics",
@@ -4679,9 +4670,16 @@ module Cnvrg
               next
             end
             uri = URI("#{prometheus_url}api/v1/query?query=#{query_content}")
-            resp = Net::HTTP.get(uri)
+            http = Net::HTTP.new(uri.host, uri.port)
+            http.use_ssl = uri.scheme == "https"
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            req = Net::HTTP::Get.new uri.request_uri
+            if prom_user.present?
+              req.basic_auth(Base64.decode64(prom_user), Base64.decode64(prom_password))
+            end
+            resp = http.request(req)
             begin
-              result = JSON.parse(resp)
+              result = JSON.parse(resp.body)
             rescue JSON::ParserError => e
               log_error(e)
               next
@@ -4690,13 +4688,22 @@ module Cnvrg
             next unless data_result
 
             if data_result.size > 1
-              stats[query_name] = {}
+              stats[query_name] = {} unless query_name.include? 'block'
               data_result.each_with_index do |res, i|
                 timestamp, value = res["value"]
                 uuid = res["metric"]["UUID"].presence || i
                 stat_value = value.present? ? ("%.2f" % value) : 0 # converting 34.685929244444445 to 34.69
                 stat_value = stat_value.to_i == stat_value.to_f ? stat_value.to_i : stat_value.to_f # converting 34.00 to 34
-                stats[query_name][uuid] = stat_value
+                if query_name.include? 'block'
+                  uuid = res["metric"]["interface"].presence || i
+                  uuid = "#{name}-#{uuid}" if name.present?
+                  stats['block_io'] = {} if stats['block_io'].blank?
+                  io_type = query_name.split('_')[1]
+                  stats['block_io'][io_type] = {} if stats['block_io'][io_type].blank?
+                  stats['block_io'][io_type].merge!({ uuid => stat_value })
+                else
+                  stats[query_name][uuid] = stat_value
+                end
               end
             else
               timestamp, value = data_result&.first&.dig('value')
@@ -4705,9 +4712,14 @@ module Cnvrg
               if query_name.include? 'block'
                 stats['block_io'] = {} if stats['block_io'].blank?
                 io_type = query_name.split('_')[1]
-                stats['block_io'].merge!({ io_type => stat_value })
+                if name.present?
+                  stats['block_io'][io_type] = {} if stats['block_io'][io_type].blank?
+                  stats['block_io'][io_type].merge!({ name => stat_value })
+                else
+                  stats['block_io'].merge!({ io_type => stat_value })
+                end
               else
-                stats[query_name] = stat_value
+                stats[query_name] = name.present? ? { name => stat_value } : stat_value
               end
             end
           end
@@ -4751,7 +4763,7 @@ module Cnvrg
     end
 
 
-    desc '', ''
+    desc '', '', :hide => true
 
     def download_built_image(image_name, image_slug)
       begin
@@ -4995,7 +5007,7 @@ module Cnvrg
       end
     end
 
-    desc 'experiments', 'List project experiments'
+    desc 'experiments', 'List project experiments', :hide => true
     method_option :id, :type => :string, :aliases => ["--id"], :desc => "Get info for specific experiments", :default => ""
     method_option :tag, :type => :string, :aliases => ["-t"], :desc => "Get info for specific experiment tag", :default => ""
 

data/lib/cnvrg/connect_job_ssh.rb

@@ -10,20 +10,20 @@ module Cnvrg
       Cnvrg::Logger.log_info("cnvrg is not configured")
     end
 
-    def start(username, password)
-      Cnvrg::API_V2.request("#{@owner}/job_ssh/#{@job_id}/start" , 'POST', {username: username, password: password})
+    def start(username, password, no_auth, port: nil)
+      Cnvrg::API_V2.request("#{@owner}/job_ssh/#{@job_id}/start" , 'POST', {username: username, password: password, no_auth: no_auth, port: port})
     end
 
     def status()
       Cnvrg::API_V2.request("#{@owner}/job_ssh/#{@job_id}/status" , 'GET', nil)
     end
 
-    def run_portforward_command(pod_name, port, kubeconfig, namespace)
+    def run_portforward_command(pod_name, port, kubeconfig, namespace, internal_port)
       command = "kubectl"
       if kubeconfig.present?
         command = "kubectl --kubeconfig=#{kubeconfig}"
       end
-      bashCommand = "#{command} -n #{namespace} port-forward #{pod_name} #{port}:22"
+      bashCommand = "#{command} -n #{namespace} port-forward #{pod_name} #{port}:#{internal_port}"
       puts("\nrunning command #{bashCommand}")
       `#{bashCommand}`
     end

data/lib/cnvrg/files.rb

@@ -106,7 +106,7 @@ module Cnvrg
             commit: commit_sha1
         })
         unless Cnvrg::CLI.is_response_success(resp, false)
-          raise SignalException.new("Cant upload files to the server.")
+          raise StandardError.new("unsupported character: folder name can not include / \\  * : ? \" | ")
         end
         # resolve bucket
         res = resp['result']

data/lib/cnvrg/helpers/agent.rb

@@ -18,10 +18,13 @@ class Cnvrg::Helpers::Agent
   #### params
   def initialize(executer: nil, slug: nil, command: nil, container_name: nil, send_log_interval: 60, timeout: -1, logs_regex: [], async: false, send_logs: false, files_exist: [], retries: 0, sleep_before_retry: 30, single_quotes: false, docker_user: nil, use_bash: false, **kwargs)
     @executer = executer
+    @job_id = ENV["CNVRG_JOB_ID"]
     @slug = slug
     @files_exist = files_exist
     @container_name = container_name
-    @run_in_slave = @container_name.downcase == "slave"
+    @is_new_main = !ENV["MAIN_CONTAINER_PORT"].blank?
+    @main_name = @is_new_main ? "main" : "slave"
+    @run_in_main = @container_name.downcase == @main_name
     @log_interval = send_log_interval
     # https://ruby-doc.org/stdlib-2.5.1/libdoc/timeout/rdoc/Timeout.html timeout should be 0 for running forever
     if timeout.blank? or timeout.negative?
@@ -37,18 +40,8 @@ class Cnvrg::Helpers::Agent
     @sleep_before_retry = sleep_before_retry
     @real_execution_retries = 0 ## How many times the command really executed until success
     @single_quotes = single_quotes
-    @docker_user = ""
-    @shell_type = use_bash ? "bash -l" : "sh"
-    if docker_user.present?
-      @docker_user = " --user #{docker_user}"
-    end
-    if @run_in_slave
-      if @single_quotes
-        @command = "docker exec #{@docker_user} -it #{@executer.slave_id} #{@shell_type} -c '#{@command}'"
-      else
-        @command = "docker exec #{@docker_user} -it #{@executer.slave_id} #{@shell_type} -c \"#{@command}\""
-      end
-    end
+    @docker_user = docker_user
+    @use_bash = use_bash
     @output = []
     @errors = []
     @exit_status = nil
@@ -75,19 +68,22 @@ class Cnvrg::Helpers::Agent
 
   def exec!
     log_internal("Command: #{@command} with slug: #{@slug} started!")
+    command_status = Status::FINISHED
     if @command.blank?
       @exit_status = 0
+      command_status = Status::ABORTED
     elsif should_run?
       send_logs(status: Status::STARTED)
       periodic_thread_handle = periodic_thread
       execute_command
     else
+      command_status = Status::ABORTED
       @exit_status = 127
     end
     finish_log = "Command: #{@command} with slug: #{@slug} finished"
     finish_log += " after #{@real_execution_retries} retries" if @real_execution_retries > 0
     log_internal(finish_log)
-    send_logs(exit_status: @exit_status, status: Status::FINISHED)
+    send_logs(exit_status: @exit_status, status: command_status)
     if periodic_thread_handle.present?
       periodic_thread_handle.join
     end
@@ -117,14 +113,50 @@ class Cnvrg::Helpers::Agent
     execute_command
   end
 
+  def execute_command_on_slave
+    extra_slug = (0...2).map { (65 + rand(26)).chr }.join
+    result_file = "/conf/result-#{@slug}-#{extra_slug}"
+    Timeout.timeout(@timeout) do
+      data = {cmd: @command, async: true, file_name: result_file, use_script: true, use_bash: @use_bash, use_sh: !@use_bash, docker_user: @docker_user}
+      conn = Cnvrg::Helpers::Executer.get_main_conn
+      response = conn.post('command', data.to_json)
+      if response.to_hash[:status].to_i != 200
+        @exit_status = 129
+        raise StandardError.new("Cant send command to slave")
+      end
+      t = FileWatch::Tail.new
+      filename = result_file
+      t.tail(filename)
+      t.subscribe do |path, line|
+        if line.include?("cnvrg-exit-code")
+          @exit_status = line.split("=")[1].to_i
+          break
+        end
+        if !@is_new_main
+          log_internal(line, level: LogLevel::PURE)
+        end
+        line = line.encode('utf-8', :invalid => :replace, :undef => :replace, :replace => '_')
+        @output << {log: line, timestamp: Time.now}
+      end
+    end
+  rescue Timeout::Error
+    @errors << {log: "Command timed out!", timestamp: Time.now}
+    log_internal("Command timed out!", level: LogLevel::ERROR)
+    @exit_status = 124
+  ensure
+    retry_command if @retries != 0 and @exit_status !=0
+    @exit_status
+  end
+
   def execute_command
+    return execute_command_on_slave if @run_in_main
     Timeout.timeout(@timeout) do
       PTY.spawn(@command) do |stdout, stdin, pid, stderr|
         @pid = pid
         begin
           if stdout.present?
             stdout.each do |line|
-              log_internal(line, level: LogLevel::PURE)
+              log_internal(line, level: LogLevel::INFO)
               line = line.strip.gsub(/\e\[([;\d]+)?m/, '')
               @output << {log: line, timestamp: Time.now}
             end
@@ -176,10 +208,18 @@ class Cnvrg::Helpers::Agent
   def log_internal(log, level: LogLevel::INFO)
     if level == LogLevel::PURE
       puts(log)
-    else
-      puts({log: log, level: level, timestamp: Time.now, command: @slug, machine_activity: @executer.machine_activity}.to_json)
+      STDOUT.flush
+      return
+    end
+    to_print = {message: log, level: level, timestamp: Time.now, command: @slug, machine_activity: @executer.machine_activity, job: @job_id}
+    if log.start_with?("{") and log.include?("timestamp")
+      log_json = JSON.parse(log)
+      to_print = to_print.stringify_keys.merge(log_json.stringify_keys)
     end
+    puts(to_print.to_json)
     STDOUT.flush
+  rescue => e
+    Cnvrg::Logger.log_error(e)
   end
 
   def filter_logs_by_regex(logs)
@@ -190,4 +230,4 @@ class Cnvrg::Helpers::Agent
       end
     end
   end
-end
+end

data/lib/cnvrg/helpers/executer.rb

@@ -1,6 +1,8 @@
+require "filewatch/tail"
 require 'cnvrg/helpers/agent'
 class Cnvrg::Helpers::Executer
-  attr_reader :machine_activity, :agent_id, :slave_id
+  attr_reader :machine_activity, :agent_id, :main_id
+  MAIN_CONTAINER_PORT = ENV['MAIN_CONTAINER_PORT'].try(:to_i) || 4000
 
 
   ### this class represent a machine_activity. it will poll the commands, communicate with the
@@ -9,11 +11,15 @@ class Cnvrg::Helpers::Executer
     @owner = owner
     @job_id = job_id
     @poll_every = poll_every
+    @check_main_every = 10
     @machine_activity = machine_activity
     @commands_q = Queue.new
     @files_q = Queue.new
     @agent_id = nil
-    @slave_id = nil
+    @main_id = nil
+    @main_start_time = nil
+    @is_new_main = !ENV["MAIN_CONTAINER_PORT"].blank?
+    @main_name = @is_new_main ? "main" : "slave"
   end
 
   def create_file_cmd(path, content)
@@ -37,9 +43,10 @@ class Cnvrg::Helpers::Executer
   def executer_stats
     return @stats if @stats.present?
     Cnvrg::Logger.log_info("getting containers")
-    @agent_id, @slave_id = containers
+    @agent_id, @main_id = containers
     Cnvrg::Logger.log_info("got containers")
     pod_name, node_name = get_node_and_pod_names
+    # For backwards compatibility we still call this slave stats
     @stats = {
         pod_name: pod_name,
         node_name: node_name,
@@ -53,39 +60,48 @@ class Cnvrg::Helpers::Executer
           cnvrg: Cnvrg::VERSION
         },
         slave: {
-            container_id: @slave_id,
-            workdir: run_in_slave('pwd'),
-            homedir: slave_homedir,
+            container_id: @main_id,
+            container_name: @main_name,
+            workdir: run_in_main('pwd'),
+            homedir: main_homedir,
             spark_path: spark_path,
-            user: run_in_slave( 'whoami'),
-            cnvrg: run_in_slave( 'which cnvrg'),
-            has_bash: run_in_slave( 'which bash'),
-            user_id: run_in_slave( 'id -u'),
-            group_id: run_in_slave( 'id -g'),
-            python_version: run_in_slave( 'python --version'),
-            python3_version: run_in_slave( 'python3 --version'),
-            pip_version: run_in_slave( 'pip --version'),
-            pip3_version: run_in_slave( 'pip3 --version')
+            user: run_in_main( 'whoami'),
+            cnvrg: run_in_main( 'which cnvrg'),
+            has_bash: run_in_main( 'which bash'),
+            user_id: run_in_main( 'id -u'),
+            group_id: run_in_main( 'id -g'),
+            python_version: run_in_main( 'python --version'),
+            python3_version: run_in_main( 'python3 --version'),
+            pip_version: run_in_main( 'pip --version'),
+            pip3_version: run_in_main( 'pip3 --version')
         },
     }
+
     @stats
   end
 
   def containers
     agent_id = nil
-    slave_id = nil
-    while agent_id.blank? or slave_id.blank?
-      grep_by = @job_id
-      grep_by = "$(hostname)" if ENV['KUBERNETES_PORT'].present?
-      cntrs = `docker ps --format "table {{.ID}},{{.Names}}" | grep -i #{grep_by}`.split("\n").map{|x| x.strip}
-      agent_id = cntrs.find{|container_name| container_name.include? "agent"}.split(",").first rescue nil
-      slave_id = cntrs.find{|container_name| container_name.include? "slave"}.split(",").first rescue nil
-      sleep(5)
+    main_id = nil
+    timeout = 2
+    timeout = nil if !@is_new_main
+    Timeout.timeout(timeout) do
+      while agent_id.blank? or main_id.blank?
+        grep_by = @job_id
+        grep_by = "$(hostname)" if ENV['KUBERNETES_PORT'].present?
+        cntrs = `docker ps --format "table {{.ID}},{{.Names}}" | grep -i #{grep_by}`.split("\n").map{|x| x.strip}
+        agent_id = cntrs.find{|container_name| container_name.include? "agent"}.split(",").first rescue nil
+        main_id = cntrs.find{|container_name| container_name.include? @main_name}.split(",").first rescue nil
+        sleep(2)
+      end
     end
-    if slave_id.blank?
-      raise "Can't find slave id"
+    if main_id.blank?
+      raise "Can't find main id"
     end
-    [agent_id, slave_id]
+    [agent_id, main_id]
+  rescue => e
+    Cnvrg::Logger.log_error(e)
+    [agent_id, main_id]
   end
 
   def current_homedir
@@ -93,21 +109,45 @@ class Cnvrg::Helpers::Executer
   end
 
   def spark_path
-    run_in_slave("env | grep SPARK_HOME").strip.split("=").try(:last)
+    run_in_main("env | grep SPARK_HOME").strip.split("=").try(:last)
   end
 
-  def slave_homedir()
-    run_in_slave("env | grep -w HOME").split("=").try(:last)
+  def main_homedir()
+    run_in_main("env | grep -w HOME").split("=").try(:last)
   end
 
-  def slave_env
-    run_in_slave("env").split("\n").map{|x| x.split("=")}
+  def main_env
+    run_in_main("env").split("\n").map{|x| x.split("=")}
   end
 
-  def run_in_slave(command)
-    `docker exec -i #{@slave_id} sh -c '#{command}'`.strip
-  end
+  def run_in_main(command)
+    data = {cmd: command, async: false, use_sh: true}
 
+    conn = Cnvrg::Helpers::Executer.get_main_conn(timeout: 180)
+    response = conn.post('command', data.to_json)
+    if response.to_hash[:status].to_i != 200
+      Cnvrg::Logger.log_info("Got back bad status #{response.to_hash[:status]}")
+      return ""
+    end
+    resp = []
+    lines = response.body.split("\n")
+    lines.each do |line|
+      next if line.strip == nil or line.strip == ""
+      if line.include?("cnvrg-exit-code")
+        exit_status = line.split("=")[1].to_i
+        if exit_status != 0
+          Cnvrg::Logger.log_info("failed to run find command #{command} on main")
+          return ""
+        end
+        next
+      end
+      resp << line
+    end
+    return resp.join("\n")
+  rescue => e
+    Cnvrg::Logger.log_error(e)
+    return ""
+  end
 
   def poll
     resp = Cnvrg::API.request([activity_url, "commands"].join('/'), "POST")
@@ -124,6 +164,7 @@ class Cnvrg::Helpers::Executer
     success = false
     puts("Agent started, connecting to #{Cnvrg::API.get_api}")
     STDOUT.flush
+    wait_for_main
     while !success and retries < 100
       begin
         resp = Cnvrg::API.request(activity_url, "PUT", {stats: executer_stats})
@@ -154,14 +195,75 @@ class Cnvrg::Helpers::Executer
     end
   end
 
+  def check_main_is_working_thread
+    while true
+      check_main_alive
+      sleep(@check_main_every)
+    end
+  end
+
   def main_thread
     init
     Thread.new do
       polling_thread
     end
+    Thread.new do
+      check_main_is_working_thread
+    end
     execute_cmds
   end
 
+  def wait_for_main
+    copy_file_to_main
+    start_tiny_if_missing
+    puts("Waiting for main container")
+    STDOUT.flush
+    got_response = false
+    while !got_response do
+      begin
+        conn = Cnvrg::Helpers::Executer.get_main_conn
+        response = conn.get('readiness')
+        if response.to_hash[:status].to_i != 200
+          sleep(0.1)
+          next
+        else
+          puts("Client container is ready")
+          STDOUT.flush
+          @main_start_time = response.body.to_i
+          got_response = true
+        end
+      rescue => e
+        puts("Failed to connect to main")
+        puts(e)
+        STDOUT.flush
+        sleep(0.1)
+        next
+      end
+    end
+  end
+
+  def copy_file_to_main
+    begin
+      FileUtils.cp("/cnvrg-tiny", "/conf/tiny")
+      FileUtils.cp_r("/scripts", "/conf/scripts-bin")
+      FileUtils.touch("/conf/tiny-ready")
+    rescue => e
+      Cnvrg::Logger.log_error(e)
+    end
+  end
+
+  def start_tiny_if_missing
+    return unless ENV['MAIN_CONTAINER_PORT'].blank?
+    Cnvrg::Logger.log_info("Tiny not found, starting it")
+    @agent_id, @main_id = containers
+    pid = Process.fork do
+      Cnvrg::Logger.log_info("running docker exec -i #{@main_id} sh -c '/conf/tiny")
+      `docker exec -i #{@main_id} sh -c '/conf/tiny'`.strip
+    end
+    Process.detach(pid)
+    Cnvrg::Logger.log_info("Tiny started and detached")
+  end
+
   def execute_cmds
     pids = []
     while true
@@ -215,6 +317,23 @@ class Cnvrg::Helpers::Executer
     Cnvrg::API.request([activity_url, "job_events"].join('/'), "POST", {pod_events: pod_events, node_events: node_events})
   end
 
+  def check_main_alive
+    # Dont check before we got first response
+    return if @main_start_time == nil
+    conn = Cnvrg::Helpers::Executer.get_main_conn
+    response = conn.get('readiness')
+    if response.to_hash[:status].to_i != 200
+      main_start_time = 0
+    else
+      main_start_time = response.body.to_i
+    end
+    if main_start_time != @main_start_time
+      puts("Found that main restarted, restarting agent")
+      Cnvrg::Logger.log_info("Found that main restarted, restarting agent")
+      exit(1)
+    end
+  end
+
   def get_pod_events(pod_name)
     return if pod_name.blank?
     `kubectl get event --field-selector involvedObject.name=#{pod_name} -o json`
@@ -224,4 +343,27 @@ class Cnvrg::Helpers::Executer
     return if node_name.blank?
     `kubectl get event --all-namespaces --field-selector involvedObject.name=#{node_name} -o json`
   end
+
+  def self.main_container_url
+    if ENV["CNVRG_COMPUTE_CLUSTER"].blank? and ENV["KUBERNETES_SERVICE_HOST"].blank?
+      if ENV["MAIN_CONTAINER_PORT"].blank?
+        host = "slave"
+      else
+        host = "main"
+      end
+      "http://#{host}:#{Cnvrg::Helpers::Executer::MAIN_CONTAINER_PORT}"
+    else
+      "http://localhost:#{Cnvrg::Helpers::Executer::MAIN_CONTAINER_PORT}"
+    end
+  end
+
+  def self.get_main_conn(timeout: 4, open_timeout: 1)
+    conn = Faraday.new(
+      url: Cnvrg::Helpers::Executer.main_container_url,
+      headers: {'Content-Type' => 'application/json'}
+    )
+    conn.options.timeout = timeout
+    conn.options.open_timeout = open_timeout
+    conn
+  end
 end

data/lib/cnvrg/job_ssh.rb

@@ -5,14 +5,18 @@ module Cnvrg
     method_option :port, :type => :numeric, :aliases => ["-p", "--port"], :desc => "Port to bind into", :default => 2222
     method_option :username, :type => :string, :aliases => ["-u", "--username"], :desc => "Job container user name", :default => nil
     method_option :password, :type => :string, :aliases => ["--password"], :desc =>"Job Conatainer user name, will be set by cnvrg", :default => nil
+    method_option :no_auth, :type => :boolean, :aliases => ["--no-auth"], :default => false
+    method_option :internal_port, :type => :numeric, :aliases => ["--internal-port"], :desc =>"Internal port in the pod for the ssh", :default => 22
     method_option :kubeconfig, :type => :string, :aliases => ["--kubeconfig"], :desc => "Path to kubeconfig, if blank default config will be used", :default => nil
     def start(job_id)
+      no_auth = options["no_auth"]
       Cnvrg::CLI.new.log_start(__method__, args, options)
       @job_ssh = ConnectJobSsh.new(job_id)
-      @job_ssh.start(options['username'], options['password'])
+      @job_ssh.start(options['username'], options['password'], no_auth, port: options['internal_port'])
       pod_name = nil
       namespace = "cnvrg"
       ssh_ready = false
+      internal_port = options['internal_port']
       while not ssh_ready
         resp = @job_ssh.status()
         status = resp["ssh_status"]
@@ -26,13 +30,14 @@ module Cnvrg
             username = resp["username"]
             pod_name = resp["pod_name"]
             namespace = resp["namespace"]
+            internal_port = resp["port"] || internal_port
             ssh_ready = true
         else
           puts("Failed to start ssh")
           break
         end 
       end
-      if pod_name.blank? or password.blank? or username.blank?
+      if pod_name.blank? or (password.blank? and !no_auth) or username.blank?
         puts("Failed to get required params")
         return
       end
@@ -41,8 +46,8 @@ module Cnvrg
       puts("host: 127.0.0.1")
       puts("port: #{options["port"]}")
       puts("username: #{username}")
-      puts("password: #{password}")
-      @job_ssh.run_portforward_command(pod_name, options["port"], options["kubeconfig"], namespace)
+      puts("password: #{password}") unless no_auth
+      @job_ssh.run_portforward_command(pod_name, options["port"], options["kubeconfig"], namespace, internal_port)
     end
   end
 end

data/lib/cnvrg/project.rb

@@ -378,14 +378,16 @@ module Cnvrg
       []
     end
 
-    def generate_output_dir(output_dir)
+    def generate_output_dir(output_dir, local: false)
       Cnvrg::Logger.log_info("Generating output dir for #{output_dir}")
       upload_list = []
       list = Dir.glob("/cnvrg/#{output_dir}/**/*", File::FNM_DOTMATCH)
+      if local
+        list += Dir.glob("#{output_dir}/**/*", File::FNM_DOTMATCH)
+      end
       Parallel.map(list, in_threads: IDXParallelThreads) do |e|
         next if e.end_with? "/."
         if File.directory? e
-
           upload_list << e + "/"
         else
           upload_list << e

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.11.31'
-end
+  VERSION = '2.0.1'
+end

metadata

@@ -1,15 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.11.31
+  version: 2.0.1
 platform: ruby
 authors:
 - Yochay Ettun
 - Leah Kolben
+- Omer Shacham
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-04 00:00:00.000000000 Z
+date: 2021-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -321,6 +322,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.1
+- !ruby/object:Gem::Dependency
+  name: filewatch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement