cnvrg 1.11.28 → 2.0.6

data/lib/cnvrg/connect_job_ssh.rb

@@ -10,20 +10,20 @@ module Cnvrg
       Cnvrg::Logger.log_info("cnvrg is not configured")
     end
 
-    def start(username, password)
-      Cnvrg::API_V2.request("#{@owner}/job_ssh/#{@job_id}/start" , 'POST', {username: username, password: password})
+    def start(username, password, no_auth, port: nil)
+      Cnvrg::API_V2.request("#{@owner}/job_ssh/#{@job_id}/start" , 'POST', {username: username, password: password, no_auth: no_auth, port: port})
     end
 
     def status()
       Cnvrg::API_V2.request("#{@owner}/job_ssh/#{@job_id}/status" , 'GET', nil)
     end
 
-    def run_portforward_command(pod_name, port, kubeconfig, namespace)
+    def run_portforward_command(pod_name, port, kubeconfig, namespace, internal_port)
       command = "kubectl"
       if kubeconfig.present?
         command = "kubectl --kubeconfig=#{kubeconfig}"
       end
-      bashCommand = "#{command} -n #{namespace} port-forward #{pod_name} #{port}:22"
+      bashCommand = "#{command} -n #{namespace} port-forward #{pod_name} #{port}:#{internal_port}"
       puts("\nrunning command #{bashCommand}")
       `#{bashCommand}`
     end

data/lib/cnvrg/experiment.rb

@@ -133,23 +133,30 @@ module Cnvrg
       return response
     end
     def remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options=0,data_query=nil, image = nil, datasets = nil)
-      response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/notebook/remote", 'POST', {instance_type: instance_type,dataset_slug:data,
-                                                                                                          dataset_commit: data_commit,image_slug:image,
-                                                                                                          datasets: datasets,
-                                                                                                          commit:commit,notebook_type:notebook_type,dataset_sync_options:ds_sync_options,
-                                                                                                          dataset_query:data_query})
+      response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/notebook/remote", 'POST', { instance_type: instance_type, dataset_slug:data,
+                                                                                                           dataset_commit: data_commit, image_slug:image,
+                                                                                                           datasets: datasets,
+                                                                                                           commit:commit, notebook_type:notebook_type, dataset_sync_options:ds_sync_options,
+                                                                                                           dataset_query: data_query })
       return response
     end
 
     def upload_temp_log(temp_log)
-      response = Cnvrg::API.request(@base_resource + "experiment/upload_temp_log", 'POST', {output: temp_log,
-                                                                                            exp_slug: @slug})
-      Cnvrg::CLI.is_response_success(response,false)
+      response = Cnvrg::API.request(@base_resource + "experiment/upload_temp_log", 'POST', { output: temp_log,
+                                                                                             exp_slug: @slug })
+      Cnvrg::CLI.is_response_success(response, false)
     end
 
     def send_machine_stats(stats)
-      response = Cnvrg::API.request(@base_resource + "experiment/upload_stats", "POST", {exp_slug: @slug, stats: stats.map{|s| s.merge!({time: Time.now})}})
-      Cnvrg::CLI.is_response_success(response,false)
+      response = Cnvrg::API.request(
+        @base_resource + "experiment/upload_stats",
+        "POST",
+        {
+          exp_slug: @slug,
+          stats: stats.map { |s| s.merge!({ time: Time.now }) }
+        }
+      )
+      Cnvrg::CLI.is_response_success(response, false)
     end
 
     def end(output, exit_status, end_commit, cpu_average, memory_average, end_time: nil)
@@ -158,7 +165,7 @@ module Cnvrg
       success = false
       end_time ||= Time.now
       while tries < 10 and success.blank?
-        sleep (tries*rand) ** 2 ### exponential backoff
+        sleep (tries * rand) ** 2 ### exponential backoff
                                 ## this call is super important so we cant let it crash.
 
         tries += 1

data/lib/cnvrg/files.rb

@@ -106,7 +106,7 @@ module Cnvrg
             commit: commit_sha1
         })
         unless Cnvrg::CLI.is_response_success(resp, false)
-          raise SignalException.new("Cant upload files to the server.")
+          raise StandardError.new("unsupported character: folder name can not include / \\  * : ? \" | ")
         end
         # resolve bucket
         res = resp['result']
@@ -730,7 +730,11 @@ module Cnvrg
       end
       res = Cnvrg::API.request(@base_resource + "download_files", 'POST', {files: files, commit: commit})
       unless Cnvrg::CLI.is_response_success(res, false)
-        raise SignalException.new("Cant download files from the server.")
+        begin
+          puts(res)
+        rescue
+        end
+        raise StandardError.new("Cant download files from the server.")
       end
       self.download_multiple_files_s3(res['result'], @project_home, postfix: postfix, progress: progress, threads: threads)
     end

data/lib/cnvrg/helpers/agent.rb

@@ -18,10 +18,13 @@ class Cnvrg::Helpers::Agent
   #### params
   def initialize(executer: nil, slug: nil, command: nil, container_name: nil, send_log_interval: 60, timeout: -1, logs_regex: [], async: false, send_logs: false, files_exist: [], retries: 0, sleep_before_retry: 30, single_quotes: false, docker_user: nil, use_bash: false, **kwargs)
     @executer = executer
+    @job_id = ENV["CNVRG_JOB_ID"]
     @slug = slug
     @files_exist = files_exist
     @container_name = container_name
-    @run_in_slave = @container_name.downcase == "slave"
+    @is_new_main = !ENV["MAIN_CONTAINER_PORT"].blank?
+    @main_name = @is_new_main ? "main" : "slave"
+    @run_in_main = @container_name.downcase == @main_name
     @log_interval = send_log_interval
     # https://ruby-doc.org/stdlib-2.5.1/libdoc/timeout/rdoc/Timeout.html timeout should be 0 for running forever
     if timeout.blank? or timeout.negative?
@@ -37,18 +40,8 @@ class Cnvrg::Helpers::Agent
     @sleep_before_retry = sleep_before_retry
     @real_execution_retries = 0 ## How many times the command really executed until success
     @single_quotes = single_quotes
-    @docker_user = ""
-    @shell_type = use_bash ? "bash -l" : "sh"
-    if docker_user.present?
-      @docker_user = " --user #{docker_user}"
-    end
-    if @run_in_slave
-      if @single_quotes
-        @command = "docker exec #{@docker_user} -it #{@executer.slave_id} #{@shell_type} -c '#{@command}'"
-      else
-        @command = "docker exec #{@docker_user} -it #{@executer.slave_id} #{@shell_type} -c \"#{@command}\""
-      end
-    end
+    @docker_user = docker_user
+    @use_bash = use_bash
     @output = []
     @errors = []
     @exit_status = nil
@@ -75,19 +68,22 @@ class Cnvrg::Helpers::Agent
 
   def exec!
     log_internal("Command: #{@command} with slug: #{@slug} started!")
+    command_status = Status::FINISHED
     if @command.blank?
       @exit_status = 0
+      command_status = Status::ABORTED
     elsif should_run?
       send_logs(status: Status::STARTED)
       periodic_thread_handle = periodic_thread
       execute_command
     else
+      command_status = Status::ABORTED
       @exit_status = 127
     end
     finish_log = "Command: #{@command} with slug: #{@slug} finished"
     finish_log += " after #{@real_execution_retries} retries" if @real_execution_retries > 0
     log_internal(finish_log)
-    send_logs(exit_status: @exit_status, status: Status::FINISHED)
+    send_logs(exit_status: @exit_status, status: command_status)
     if periodic_thread_handle.present?
       periodic_thread_handle.join
     end
@@ -117,14 +113,50 @@ class Cnvrg::Helpers::Agent
     execute_command
   end
 
+  def execute_command_on_slave
+    extra_slug = (0...2).map { (65 + rand(26)).chr }.join
+    result_file = "/conf/result-#{@slug}-#{extra_slug}"
+    Timeout.timeout(@timeout) do
+      data = {cmd: @command, async: true, file_name: result_file, use_script: true, use_bash: @use_bash, use_sh: !@use_bash, docker_user: @docker_user}
+      conn = Cnvrg::Helpers::Executer.get_main_conn
+      response = conn.post('command', data.to_json)
+      if response.to_hash[:status].to_i != 200
+        @exit_status = 129
+        raise StandardError.new("Cant send command to slave")
+      end
+      t = FileWatch::Tail.new
+      filename = result_file
+      t.tail(filename)
+      t.subscribe do |path, line|
+        if line.include?("cnvrg-exit-code")
+          @exit_status = line.split("=")[1].to_i
+          break
+        end
+        if !@is_new_main
+          log_internal(line, level: LogLevel::PURE)
+        end
+        line = line.encode('utf-8', :invalid => :replace, :undef => :replace, :replace => '_')
+        @output << {log: line, timestamp: Time.now}
+      end
+    end
+  rescue Timeout::Error
+    @errors << {log: "Command timed out!", timestamp: Time.now}
+    log_internal("Command timed out!", level: LogLevel::ERROR)
+    @exit_status = 124
+  ensure
+    retry_command if @retries != 0 and @exit_status !=0
+    @exit_status
+  end
+
   def execute_command
+    return execute_command_on_slave if @run_in_main
     Timeout.timeout(@timeout) do
       PTY.spawn(@command) do |stdout, stdin, pid, stderr|
         @pid = pid
         begin
           if stdout.present?
             stdout.each do |line|
-              log_internal(line, level: LogLevel::PURE)
+              log_internal(line, level: LogLevel::INFO)
               line = line.strip.gsub(/\e\[([;\d]+)?m/, '')
               @output << {log: line, timestamp: Time.now}
             end
@@ -176,10 +208,18 @@ class Cnvrg::Helpers::Agent
   def log_internal(log, level: LogLevel::INFO)
     if level == LogLevel::PURE
       puts(log)
-    else
-      puts({log: log, level: level, timestamp: Time.now, command: @slug, machine_activity: @executer.machine_activity}.to_json)
+      STDOUT.flush
+      return
+    end
+    to_print = {message: log, level: level, timestamp: Time.now, command: @slug, machine_activity: @executer.machine_activity, job: @job_id}
+    if log.start_with?("{") and log.include?("timestamp")
+      log_json = JSON.parse(log)
+      to_print = to_print.stringify_keys.merge(log_json.stringify_keys)
     end
+    puts(to_print.to_json)
     STDOUT.flush
+  rescue => e
+    Cnvrg::Logger.log_error(e)
   end
 
   def filter_logs_by_regex(logs)
@@ -190,4 +230,4 @@ class Cnvrg::Helpers::Agent
       end
     end
   end
-end
+end

data/lib/cnvrg/helpers/executer.rb

@@ -1,7 +1,9 @@
+require "filewatch/tail"
 require 'cnvrg/helpers/agent'
 class Cnvrg::Helpers::Executer
-  attr_reader :machine_activity, :agent_id, :slave_id
-
+  attr_reader :machine_activity, :agent_id, :main_id
+  MAIN_CONTAINER_PORT = ENV['MAIN_CONTAINER_PORT'].try(:to_i) || 4000
+  HAS_DOCKER = ENV['HAS_DOCKER'] == "true"
 
   ### this class represent a machine_activity. it will poll the commands, communicate with the
   # server (poll commands) and let the server know the status of this executer.
@@ -9,11 +11,15 @@ class Cnvrg::Helpers::Executer
     @owner = owner
     @job_id = job_id
     @poll_every = poll_every
+    @check_main_every = 10
     @machine_activity = machine_activity
     @commands_q = Queue.new
     @files_q = Queue.new
     @agent_id = nil
-    @slave_id = nil
+    @main_id = nil
+    @main_start_time = nil
+    @is_new_main = !ENV["MAIN_CONTAINER_PORT"].blank?
+    @main_name = @is_new_main ? "main" : "slave"
   end
 
   def create_file_cmd(path, content)
@@ -37,9 +43,10 @@ class Cnvrg::Helpers::Executer
   def executer_stats
     return @stats if @stats.present?
     Cnvrg::Logger.log_info("getting containers")
-    @agent_id, @slave_id = containers
+    @agent_id, @main_id = containers
     Cnvrg::Logger.log_info("got containers")
     pod_name, node_name = get_node_and_pod_names
+    # For backwards compatibility we still call this slave stats
     @stats = {
         pod_name: pod_name,
         node_name: node_name,
@@ -53,39 +60,48 @@ class Cnvrg::Helpers::Executer
           cnvrg: Cnvrg::VERSION
         },
         slave: {
-            container_id: @slave_id,
-            workdir: run_in_slave('pwd'),
-            homedir: slave_homedir,
+            container_id: @main_id,
+            container_name: @main_name,
+            workdir: run_in_main('pwd'),
+            homedir: main_homedir,
             spark_path: spark_path,
-            user: run_in_slave( 'whoami'),
-            cnvrg: run_in_slave( 'which cnvrg'),
-            has_bash: run_in_slave( 'which bash'),
-            user_id: run_in_slave( 'id -u'),
-            group_id: run_in_slave( 'id -g'),
-            python_version: run_in_slave( 'python --version'),
-            python3_version: run_in_slave( 'python3 --version'),
-            pip_version: run_in_slave( 'pip --version'),
-            pip3_version: run_in_slave( 'pip3 --version')
+            user: run_in_main( 'whoami'),
+            cnvrg: run_in_main( 'which cnvrg'),
+            has_bash: run_in_main( 'which bash'),
+            user_id: run_in_main( 'id -u'),
+            group_id: run_in_main( 'id -g'),
+            python_version: run_in_main( 'python --version'),
+            python3_version: run_in_main( 'python3 --version'),
+            pip_version: run_in_main( 'pip --version'),
+            pip3_version: run_in_main( 'pip3 --version')
         },
     }
+
     @stats
   end
 
   def containers
     agent_id = nil
-    slave_id = nil
-    while agent_id.blank? or slave_id.blank?
-      grep_by = @job_id
-      grep_by = "$(hostname)" if ENV['KUBERNETES_PORT'].present?
-      cntrs = `docker ps --format "table {{.ID}},{{.Names}}" | grep -i #{grep_by}`.split("\n").map{|x| x.strip}
-      agent_id = cntrs.find{|container_name| container_name.include? "agent"}.split(",").first rescue nil
-      slave_id = cntrs.find{|container_name| container_name.include? "slave"}.split(",").first rescue nil
-      sleep(5)
+    main_id = nil
+    timeout = 2
+    timeout = nil if (!@is_new_main || HAS_DOCKER)
+    Timeout.timeout(timeout) do
+      while agent_id.blank? or main_id.blank?
+        grep_by = @job_id
+        grep_by = "$(hostname)" if ENV['KUBERNETES_PORT'].present?
+        cntrs = `docker ps --format "table {{.ID}},{{.Names}}" | grep -i #{grep_by}`.split("\n").map{|x| x.strip}
+        agent_id = cntrs.find{|container_name| container_name.include? "agent"}.split(",").first rescue nil
+        main_id = cntrs.find{|container_name| container_name.include? @main_name}.split(",").first rescue nil
+        sleep(2)
+      end
     end
-    if slave_id.blank?
-      raise "Can't find slave id"
+    if main_id.blank?
+      raise "Can't find main id"
     end
-    [agent_id, slave_id]
+    [agent_id, main_id]
+  rescue => e
+    Cnvrg::Logger.log_error(e)
+    [agent_id, main_id]
   end
 
   def current_homedir
@@ -93,21 +109,45 @@ class Cnvrg::Helpers::Executer
   end
 
   def spark_path
-    run_in_slave("env | grep SPARK_HOME").strip.split("=").try(:last)
+    run_in_main("env | grep SPARK_HOME").strip.split("=").try(:last)
   end
 
-  def slave_homedir()
-    run_in_slave("env | grep -w HOME").split("=").try(:last)
+  def main_homedir()
+    run_in_main("env | grep -w HOME").split("=").try(:last)
   end
 
-  def slave_env
-    run_in_slave("env").split("\n").map{|x| x.split("=")}
+  def main_env
+    run_in_main("env").split("\n").map{|x| x.split("=")}
   end
 
-  def run_in_slave(command)
-    `docker exec -i #{@slave_id} sh -c '#{command}'`.strip
-  end
+  def run_in_main(command)
+    data = {cmd: command, async: false, use_sh: true}
 
+    conn = Cnvrg::Helpers::Executer.get_main_conn(timeout: 180)
+    response = conn.post('command', data.to_json)
+    if response.to_hash[:status].to_i != 200
+      Cnvrg::Logger.log_info("Got back bad status #{response.to_hash[:status]}")
+      return ""
+    end
+    resp = []
+    lines = response.body.split("\n")
+    lines.each do |line|
+      next if line.strip == nil or line.strip == ""
+      if line.include?("cnvrg-exit-code")
+        exit_status = line.split("=")[1].to_i
+        if exit_status != 0
+          Cnvrg::Logger.log_info("failed to run find command #{command} on main")
+          return ""
+        end
+        next
+      end
+      resp << line
+    end
+    return resp.join("\n")
+  rescue => e
+    Cnvrg::Logger.log_error(e)
+    return ""
+  end
 
   def poll
     resp = Cnvrg::API.request([activity_url, "commands"].join('/'), "POST")
@@ -124,6 +164,7 @@ class Cnvrg::Helpers::Executer
     success = false
     puts("Agent started, connecting to #{Cnvrg::API.get_api}")
     STDOUT.flush
+    wait_for_main
     while !success and retries < 100
       begin
         resp = Cnvrg::API.request(activity_url, "PUT", {stats: executer_stats})
@@ -154,14 +195,75 @@ class Cnvrg::Helpers::Executer
     end
   end
 
+  def check_main_is_working_thread
+    while true
+      check_main_alive
+      sleep(@check_main_every)
+    end
+  end
+
   def main_thread
     init
     Thread.new do
       polling_thread
     end
+    Thread.new do
+      check_main_is_working_thread
+    end
     execute_cmds
   end
 
+  def wait_for_main
+    copy_file_to_main
+    start_tiny_if_missing
+    puts("Waiting for main container")
+    STDOUT.flush
+    got_response = false
+    while !got_response do
+      begin
+        conn = Cnvrg::Helpers::Executer.get_main_conn
+        response = conn.get('readiness')
+        if response.to_hash[:status].to_i != 200
+          sleep(0.1)
+          next
+        else
+          puts("Client container is ready")
+          STDOUT.flush
+          @main_start_time = response.body.to_i
+          got_response = true
+        end
+      rescue => e
+        puts("Failed to connect to main")
+        puts(e)
+        STDOUT.flush
+        sleep(0.1)
+        next
+      end
+    end
+  end
+
+  def copy_file_to_main
+    begin
+      FileUtils.cp("/cnvrg-tiny", "/conf/tiny")
+      FileUtils.cp_r("/scripts", "/conf/scripts-bin")
+      FileUtils.touch("/conf/tiny-ready")
+    rescue => e
+      Cnvrg::Logger.log_error(e)
+    end
+  end
+
+  def start_tiny_if_missing
+    return unless ENV['MAIN_CONTAINER_PORT'].blank?
+    Cnvrg::Logger.log_info("Tiny not found, starting it")
+    @agent_id, @main_id = containers
+    pid = Process.fork do
+      Cnvrg::Logger.log_info("running docker exec -i #{@main_id} sh -c '/conf/tiny")
+      `docker exec -i #{@main_id} sh -c '/conf/tiny'`.strip
+    end
+    Process.detach(pid)
+    Cnvrg::Logger.log_info("Tiny started and detached")
+  end
+
   def execute_cmds
     pids = []
     while true
@@ -201,7 +303,7 @@ class Cnvrg::Helpers::Executer
     pod_name = `hostname`.strip rescue nil
     node_name = nil
     if pod_name.present?
-      pod_describe = `kubectl -n cnvrg get pod #{pod_name} -o json` rescue nil
+      pod_describe = `kubectl get pod #{pod_name} -o json` rescue nil
       pod_describe = JSON.parse(pod_describe) rescue {}
       node_name = pod_describe["spec"]["nodeName"] rescue nil
     end
@@ -215,13 +317,53 @@ class Cnvrg::Helpers::Executer
     Cnvrg::API.request([activity_url, "job_events"].join('/'), "POST", {pod_events: pod_events, node_events: node_events})
   end
 
+  def check_main_alive
+    # Dont check before we got first response
+    return if @main_start_time == nil
+    conn = Cnvrg::Helpers::Executer.get_main_conn
+    response = conn.get('readiness')
+    if response.to_hash[:status].to_i != 200
+      main_start_time = 0
+    else
+      main_start_time = response.body.to_i
+    end
+    if main_start_time != @main_start_time
+      puts("Found that main restarted, restarting agent")
+      Cnvrg::Logger.log_info("Found that main restarted, restarting agent")
+      exit(1)
+    end
+  end
+
   def get_pod_events(pod_name)
     return if pod_name.blank?
-    `kubectl get event --namespace cnvrg --field-selector involvedObject.name=#{pod_name} -o json`
+    `kubectl get event --field-selector involvedObject.name=#{pod_name} -o json`
   end
 
   def get_node_events(node_name)
     return if node_name.blank?
     `kubectl get event --all-namespaces --field-selector involvedObject.name=#{node_name} -o json`
   end
+
+  def self.main_container_url
+    if ENV["CNVRG_COMPUTE_CLUSTER"].blank? and ENV["KUBERNETES_SERVICE_HOST"].blank?
+      if ENV["MAIN_CONTAINER_PORT"].blank?
+        host = "slave"
+      else
+        host = "main"
+      end
+      "http://#{host}:#{Cnvrg::Helpers::Executer::MAIN_CONTAINER_PORT}"
+    else
+      "http://localhost:#{Cnvrg::Helpers::Executer::MAIN_CONTAINER_PORT}"
+    end
+  end
+
+  def self.get_main_conn(timeout: 4, open_timeout: 1)
+    conn = Faraday.new(
+      url: Cnvrg::Helpers::Executer.main_container_url,
+      headers: {'Content-Type' => 'application/json'}
+    )
+    conn.options.timeout = timeout
+    conn.options.open_timeout = open_timeout
+    conn
+  end
 end