cnvrg 1.6.38 → 1.9.5

data/lib/cnvrg/helpers/agent.rb

@@ -0,0 +1,188 @@
+class Cnvrg::Helpers::Agent
+
+  module Status
+    STARTED = :started
+    RUNNING = :running
+    FINISHED = :finished
+    ABORTED = "aborted"
+  end
+
+  module LogLevel
+    INFO = :info
+    PURE = :pure
+    ERROR = :error
+  end
+
+  #### This class represent a single command in the system.
+  #### it runs under an executer (machine_activity) so it should have all the executer
+  #### params
+  def initialize(executer: nil, slug: nil, command: nil, container_name: nil, send_log_interval: 60, timeout: -1, logs_regex: [], async: false, send_logs: false, files_exist: [], retries: 0, sleep_before_retry: 30, single_quotes: false, docker_user: nil, use_bash: false, **kwargs)
+    @executer = executer
+    @slug = slug
+    @files_exist = files_exist
+    @container_name = container_name
+    @run_in_slave = @container_name.downcase == "slave"
+    @log_interval = send_log_interval
+    # https://ruby-doc.org/stdlib-2.5.1/libdoc/timeout/rdoc/Timeout.html timeout should be 0 for running forever
+    if timeout.blank? or timeout.negative?
+      @timeout = 0
+    else
+      @timeout = timeout
+    end
+    @logs_regex = logs_regex || []
+    @async = async
+    @command = command
+    @send_logs = send_logs
+    @retries = retries.try(:to_i) ## How many times the user asked to try to execute the command again
+    @sleep_before_retry = sleep_before_retry
+    @real_execution_retries = 0 ## How many times the command really executed until success
+    @single_quotes = single_quotes
+    @docker_user = ""
+    @shell_type = use_bash ? "bash -l" : "sh"
+    if docker_user.present?
+      @docker_user = " --user #{docker_user}"
+    end
+    if @run_in_slave
+      if @single_quotes
+        @command = "docker exec #{@docker_user} -it #{@executer.slave_id} #{@shell_type} -c '#{@command}'"
+      else
+        @command = "docker exec #{@docker_user} -it #{@executer.slave_id} #{@shell_type} -c \"#{@command}\""
+      end
+    end
+    @output = []
+    @errors = []
+    @exit_status = nil
+    @is_running = true
+    @pid = nil
+  end
+
+  def base_url
+    [@executer.activity_url, "commands", @slug].join("/")
+  end
+
+  def should_run?
+    if @files_exist.present?
+      file_doesnt_exists = @files_exist.find do |file|
+        not File.exists? file
+      end
+      return true if file_doesnt_exists.blank?
+      log_internal("Can't find file #{file_doesnt_exists}, stopping the job")
+      return false
+    end
+    true
+  end
+
+
+  def exec!
+    log_internal("Command: #{@command} with slug: #{@slug} started!")
+    if should_run?
+      send_logs(status: Status::STARTED)
+      periodic_thread
+      execute_command
+    else
+      @exit_status = 127
+    end
+    finish_log = "Command: #{@command} with slug: #{@slug} finished"
+    finish_log += " after #{@real_execution_retries} retries" if @real_execution_retries > 0
+    log_internal(finish_log)
+    send_logs(exit_status: @exit_status, status: Status::FINISHED)
+  end
+
+  def get_logs_to_send
+    new_logs = @output.pop(@output.length)
+    new_errors = @errors.pop(@errors.length)
+    [new_logs, new_errors]
+  end
+
+
+  def periodic_thread
+    Thread.new do
+        while @exit_status.blank?
+          Thread.exit if @log_interval.blank?
+          sleep(@log_interval)
+          send_logs
+        end
+    end
+  end
+
+  def retry_command
+    @retries -=1
+    sleep @sleep_before_retry
+    @real_execution_retries +=1
+    execute_command
+  end
+
+  def execute_command
+    Timeout.timeout(@timeout) do
+      PTY.spawn(@command) do |stdout, stdin, pid, stderr|
+        @pid = pid
+        begin
+          if stdout.present?
+            stdout.each do |line|
+              log_internal(line, level: LogLevel::PURE)
+              line = line.strip.gsub(/\e\[([;\d]+)?m/, '')
+              @output << {log: line, timestamp: Time.now}
+            end
+          end
+
+          if stderr.present?
+            stderr.each do |line|
+              line = line.strip.gsub(/\e\[([;\d]+)?m/, '')
+              log_internal(line, level: LogLevel::ERROR)
+              @errors << {log: line, timestamp: Time.now}
+            end
+          end
+        rescue Errno::EIO => e
+          next
+        rescue => e
+          log_internal(e.message, level: LogLevel::ERROR)
+          log_internal(e.backtrace.join("\n"), level: LogLevel::ERROR)
+          @errors << {log: e.message, timestamp: Time.now}
+        end
+        ::Process.wait pid
+      end
+    end
+    @exit_status = $?.exitstatus
+  rescue Timeout::Error
+    Process.kill(0, @pid)
+    @errors << {log: "Command timed out!", timestamp: Time.now}
+    log_internal("Command timed out!", level: LogLevel::ERROR)
+    @exit_status = 124
+  ensure
+    retry_command if @retries != 0 and @exit_status !=0
+    @exit_status
+  end
+
+  private
+  def send_logs(exit_status: nil, status: Status::RUNNING)
+    logs, error_logs = get_logs_to_send
+    # Filter logs only if not failed
+    if exit_status.blank? or exit_status == 0
+      logs = filter_logs_by_regex(logs)
+    end
+    ### there is no logs, no exit_status and status is running.
+    ### this condition let us call "send_logs" every interval iteration.
+    if logs.blank? and error_logs.blank? and exit_status.blank? and status == Status::RUNNING
+      return
+    end
+    Cnvrg::API.request(base_url, 'PUT', {logs: logs, error_logs: error_logs, exit_status: exit_status, status: status, execution_retries: @real_execution_retries, pid: @pid})
+  end
+
+  def log_internal(log, level: LogLevel::INFO)
+    if level == LogLevel::PURE
+      puts(log)
+    else
+      puts({log: log, level: level, timestamp: Time.now, command: @slug, machine_activity: @executer.machine_activity}.to_json)
+    end
+    STDOUT.flush
+  end
+
+  def filter_logs_by_regex(logs)
+    logs.select do |log|
+      next true if @send_logs
+      @logs_regex.find do |regexp_str|
+        Regexp.new(regexp_str).match(log[:log]).present?
+      end
+    end
+  end
+end

data/lib/cnvrg/helpers/executer.rb

@@ -1,309 +1,213 @@
+require 'cnvrg/helpers/agent'
 class Cnvrg::Helpers::Executer
-  module CommandsStatus
-    ACK = :ack
-    STARTED = :started
-    FAILED = :failed
-    SUCCESS = :success
-  end
-  def initialize(project: nil, job_type: nil, job_id: nil, image: nil)
-    @image = image
-    @project = project || Cnvrg::Project.new(owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
-    @job_type = job_type || ENV['CNVRG_JOB_TYPE']
-    @job_id = job_id || ENV['CNVRG_JOB_ID']
-    if @job_id.blank?
-      Cnvrg::CLI.log_message("Cant find job, exiting.", 'red')
-      exit(1)
-    end
-  end
+  attr_reader :machine_activity, :agent_id, :slave_id
 
 
-  def self.get_executer
-    if ENV["CNVRG_JOB_ID"].blank?
-      return
-    end
-    self.new
-  end
-
-  def fetch_commands(block: nil, key: nil)
-    resp = Cnvrg::API.request("#{base_url}/commands", "GET", {block: block})
-    commands = decrypt_commands(resp["commands"], resp["key"], resp["iv"])
-    commands.map{|k| k.with_indifferent_access}
+  ### this class represent a machine_activity. it will poll the commands, communicate with the
+  # server (poll commands) and let the server know the status of this executer.
+  def initialize(owner: nil, machine_activity: nil, poll_every: 30, job_id: nil)
+    @owner = owner
+    @job_id = job_id
+    @poll_every = poll_every
+    @machine_activity = machine_activity
+    @commands_q = Queue.new
+    @files_q = Queue.new
+    @agent_id = nil
+    @slave_id = nil
   end
 
-  def decrypt_commands(text,key,iv)
-    text, key, iv = [text,key,iv].map{|x| x.unpack('m')[0]}
-    decipher = OpenSSL::Cipher::AES256.new :CBC
-    decipher.decrypt
-    decipher.key = key
-    decipher.iv = iv
-    commands = decipher.update(text) + decipher.final
-    JSON.parse(commands)
-  end
-
-  def execute(cmd)
-    ## execute the command for running
-    # cmd will have to following fields
-    #
-    # :command => the command to execute
-    # :type => the command type, 'notify' or nil
-    # :timeout => the timeout for the command in seconds (default is 60 hours)
-    # :retries => integer, default 1
-    #
-    # when type == 'file_exists'
-    #  'file' => string => file to check (fullpath)
-    #  'exists_commands' => list of commands in case file exists
-    #   'non_exists_commands' => list of commands in case file doesnt exists
-    # when type == 'notify'
-    # :before_execute_log => log to be logged before execution
-    # :logs => boolean => add the execution logs to the job logs
-    # :title => command title, can replace the on_error, on_success fields
-    # :on_error_log => log to be logged on exit_code != 0
-    # :on_success_log => log to be logged on exit_code == 0
-    # when type == 'notify_command'
-    # notify to commands api about command progress
-    # when type == 'spawn'
-    # run in another process and detach from it
-    #
-    retries = cmd[:retries] || 1
-    resp = []
-    retries.times.each do
-      resp = execute_helper(cmd)
-      exit_status, _, _, _, _ = resp
-      return resp if exit_status == 0
+  def create_file_cmd(path, content)
+    if path.include? "~"
+      path = File.expand_path(path)
     end
-    return resp
+    FileUtils.mkdir_p(File.dirname(path))
+    File.open(path, "w+"){|f| f.write(content)}
   end
 
-  def execute_cmds(cmds)
-    cmds.each do |command|
-      puts "===================="
-      if command[:type] == "file_exists"
-        puts "Looking for file #{command[:file]}"
-      else
-        puts "Execute #{command[:command]}" unless command[:no_stdout]
-      end
-      execute(command)
+  def handle_files(files)
+    (files || {}).each do |path, content|
+      create_file_cmd(path, content)
     end
   end
 
-  def get_requirements_commands
-    resp = Cnvrg::API.request("#{base_url}/requirements", "GET")
-    commands = resp["commands"]
-    commands.map{|k| k.with_indifferent_access}
+  def activity_url
+    ['users', @owner, 'machine_activities', @machine_activity].join("/")
+  end
+
+  def executer_stats
+    return @stats if @stats.present?
+    Cnvrg::Logger.log_info("getting containers")
+    @agent_id, @slave_id = containers
+    Cnvrg::Logger.log_info("got containers")
+    pod_name, node_name = get_node_and_pod_names
+    @stats = {
+        pod_name: pod_name,
+        node_name: node_name,
+        agent: {
+          container_id: @agent_id,
+          workdir: `pwd`.strip,
+          homedir: current_homedir,
+          user: `whoami`.strip,
+          user_id: `id -u`.strip,
+          group_id: `id -g`.strip,
+          cnvrg: Cnvrg::VERSION
+        },
+        slave: {
+            container_id: @slave_id,
+            workdir: run_in_slave('pwd'),
+            homedir: slave_homedir,
+            spark_path: spark_path,
+            user: run_in_slave( 'whoami'),
+            cnvrg: run_in_slave( 'which cnvrg'),
+            has_bash: run_in_slave( 'which bash'),
+            user_id: run_in_slave( 'id -u'),
+            group_id: run_in_slave( 'id -g'),
+            python_version: run_in_slave( 'python --version'),
+            python3_version: run_in_slave( 'python3 --version'),
+            pip_version: run_in_slave( 'pip --version'),
+            pip3_version: run_in_slave( 'pip3 --version')
+        },
+    }
+    @stats
+  end
+
+  def containers
+    agent_id = nil
+    slave_id = nil
+    while agent_id.blank? or slave_id.blank?
+      grep_by = @job_id
+      grep_by = "$(hostname)" if ENV['KUBERNETES_PORT'].present?
+      cntrs = `docker ps --format "table {{.ID}},{{.Names}}" | grep -i #{grep_by}`.split("\n").map{|x| x.strip}
+      agent_id = cntrs.find{|container_name| container_name.include? "agent"}.split(",").first rescue nil
+      slave_id = cntrs.find{|container_name| container_name.include? "slave"}.split(",").first rescue nil
+      sleep(5)
+    end
+    if slave_id.blank?
+      raise "Can't find slave id"
+    end
+    [agent_id, slave_id]
   end
 
-  def get_commands
-    resp = Cnvrg::API.request("#{base_url}/commands", "GET")
-    commands = resp["commands"]
-    commands.map{|k| k.with_indifferent_access}
+  def current_homedir
+    `env | grep HOME`.strip.split("=").try(:last)
   end
 
-  def get_command(command_slug)
-    resp = Cnvrg::API.request("#{base_url}/commands/#{command_slug}", "GET")
-    command = resp["command"]
-    command.with_indifferent_access
+  def spark_path
+    run_in_slave("env | grep SPARK_HOME").strip.split("=").try(:last)
   end
 
-  def update_command(status, context, command_slug)
-    Cnvrg::API.request("#{base_url}/commands/#{command_slug}", "PUT", {status: status, context: context, timestamp: Time.now})
+  def slave_homedir()
+    run_in_slave("env | grep HOME").split("=").try(:last)
   end
 
-
-  def update_git_commit
-    git_commit = `git rev-parse --verify HEAD`
-    return if git_commit.blank?
-    Cnvrg::API.request("#{base_url}/update_git_commit", "POST", {git_commit: git_commit.strip!})
+  def slave_env
+    run_in_slave("env").split("\n").map{|x| x.split("=")}
   end
 
-  def set_dataset_status(dataset: nil, status: nil)
-    Cnvrg::API.request("#{base_url}/datasets/#{dataset}", "PUT", {status: status})
+  def run_in_slave(command)
+    `docker exec -i #{@slave_id} sh -c '#{command}'`.strip
   end
 
-  def monitor_command(command, command_slug)
-    monitor_single_command(command, command_slug)
-  end
 
-  private
-  def execute_helper(cmd)
-    case cmd[:type]
-    when 'notify'
-      return run_and_notify(cmd)
-    when 'notify_command'
-      return notify_command(cmd)
-    when 'spawn'
-      return spawn_command(cmd)
-    when 'file_exists'
-      if File.exists? cmd[:file]
-        return execute_cmds(cmd[:exists_commands]) if cmd[:exists_commands].present?
-      else
-        return execute_cmds(cmd[:non_exists_commands]) if cmd[:non_exists_commands].present?
-      end
-    when 'create_file'
-      return create_file(cmd)
-    else
-      return regular_command(cmd)
+  def poll
+    resp = Cnvrg::API.request([activity_url, "commands"].join('/'), "POST")
+    commands = resp["commands"]
+    files = resp["files"]
+    handle_files(files)
+    commands.each{|cmd| @commands_q.push(cmd)}
+  rescue => e
+    Cnvrg::Logger.log_error(e)
+  end
+
+  def init
+    resp = Cnvrg::API.request(activity_url, "PUT", {stats: executer_stats})
+    machine_activity = resp["machine_activity"]
+    Cnvrg::Logger.log_info("Got back machine activity #{machine_activity}")
+    if machine_activity.present? and @machine_activity != machine_activity
+      Cnvrg::Logger.log_info("Changing to machine activity #{machine_activity}")
+      machine_activity_yml = {slug: machine_activity}
+      File.open("/conf/.machine_activity.yml", "w+") {|f| f.write machine_activity_yml.to_yaml}
+      @machine_activity = machine_activity
     end
+  rescue => e
+    Cnvrg::Logger.log_error(e)
   end
 
-  def notify_command(cmd)
-    exit_status, output, errors, start_time, end_time = regular_command(cmd)
-    context ={
-        exit_status: exit_status,
-        output: output,
-        errors: errors,
-        start_time: start_time,
-        end_time: end_time
-    }
-    if exit_status == 0
-      update_command(CommandsStatus::SUCCESS, context, cmd[:command_slug])
-    else
-      update_command(CommandsStatus::FAILED, context, cmd[:command_slug])
+  def polling_thread
+    while true
+      poll
+      sleep(@poll_every)
     end
   end
 
-  def run_and_notify(cmd)
-    with_logs = cmd[:logs]
-    cmd = init_cmd_logs(cmd)
-    job_log(cmd[:before_execute_log]) if cmd[:before_execute_log].present?
-    exit_status, output, errors, start_time, end_time = regular_command(cmd)
-    logs = []
-    if exit_status == 0
-      logs = output.map{|log| log[:logs]} if with_logs
-      job_log(logs + cmd[:on_success_log])
-    else
-      logs = output + errors
-      logs = logs.sort_by{|x| x[:timestamp]}.map{|x| x[:logs]} if with_logs
-      job_log(logs + cmd[:on_error_log], level: 'error')
+  def main_thread
+    init
+    Thread.new do
+      polling_thread
     end
-    return [exit_status, output, errors, start_time, end_time]
-  end
-
-  def merge_log_block(logs)
-    logs.group_by {|log| log[:timestamp].to_s}
-        .map {|ts, logz| {timestamp: ts, logs: logz.map {|l| l[:log]}.join("\n")}}
+    execute_cmds
   end
 
+  def execute_cmds
+    pids = []
+    while true
+      if @commands_q.empty?
+        sleep(5)
+        next
+      end
+      cmd = @commands_q.pop.symbolize_keys
+      command_json = Cnvrg::API.request([activity_url, "commands", cmd[:slug]].join('/'), "GET")
 
-  def spawn_command(cmd={})
-    pid = Process.spawn(cmd[:command])
-    Process.detach(pid)
-  end
+      cmd_status = command_json["status"] rescue ""
 
-  def regular_command(cmd = {})
-    errors = []
-    output = []
-    start_time = Time.now
-    timeout = cmd[:timeout] || 5*60
-    exit_status = nil
-    t = Thread.new do
-      PTY.spawn(cmd[:command]) do |stdout, stdin, pid, stderr|
-        begin
-          if stdout.present?
-            stdout.each do |line|
-              puts line
-              output << {log: line.strip, timestamp: Time.now}
-            end
-          end
-          if stderr.present?
-            stderr.each do |line|
-              errors << {log: line.strip, timestamp: Time.now}
-            end
-          end
-        rescue Errno::EIO
-        rescue => e
-          errors << {log: e.message, timestamp: Time.now}
-        end
-        ::Process.wait pid
+      if cmd_status == Cnvrg::Helpers::Agent::Status::ABORTED
+        Cnvrg::Logger.log_info("stopping job because command #{cmd[:slug]} with status #{cmd_status}")
+        next
       end
-      exit_status = $?.exitstatus
-    end
-    while t.status
-      if Time.now - start_time > timeout
-        puts "Kill thread because of timeout..."
-        errors << {log: "Timeout", timestamp: Time.now}
-        Thread.kill(t)
-        exit_status = 100 ##killed
+      pid = Process.fork do
+        Cnvrg::Helpers::Agent.new(executer: self, **cmd).exec!
+      end
+      if cmd[:async].blank?
+        Process.waitpid(pid)
+      else
+        Process.detach(pid)
       end
-      sleep 1
+      pids << pid
+      ######
     end
-    end_time = Time.now
-    [exit_status, merge_log_block(output), merge_log_block(errors), start_time, end_time]
+    pids
   end
 
-  def create_file(cmd)
-    cmd = init_cmd_logs(cmd)
-    File.open(cmd[:path], "w+"){|f| f.write(cmd[:content])}
+  def merge_log_block(logs)
+    logs.group_by {|log| log[:timestamp].to_s}
+        .map {|ts, logz| {timestamp: ts, logs: logz.map {|l| l[:log]}.join("\n")}}
   end
 
-  def init_cmd_logs(cmd)
-    if cmd[:title]
-      cmd[:before_execute_log] ||= ["Running #{cmd[:title]}"]
-      cmd[:on_error_log] ||= ["#{cmd[:title]} was failed during running"]
-      cmd[:on_success_log] ||= ["#{cmd[:title]} executed successfully"]
+  def get_node_and_pod_names
+    pod_name = `hostname`.strip rescue nil
+    node_name = nil
+    if pod_name.present?
+      pod_describe = `kubectl -n cnvrg get pod #{pod_name} -o json` rescue nil
+      pod_describe = JSON.parse(pod_describe) rescue {}
+      node_name = pod_describe["spec"]["nodeName"] rescue nil
     end
-    cmd[:on_success_log] ||= []
-    cmd[:on_error_log] ||= []
-    [:before_execute_log, :on_success_log, :on_error_log].each{|x| cmd[x] = [cmd[x]].flatten}
-    cmd
+    [pod_name, node_name]
   end
 
-  def base_url
-    "users/#{@project.owner}/projects/#{@project.slug}/jobs/#{@job_type.underscore}/#{@job_id}"
+  def pre_pod_stop
+    pod_name, node_name = get_node_and_pod_names
+    pod_events = get_pod_events(pod_name)
+    node_events = get_node_events(node_name)
+    Cnvrg::API.request([activity_url, "job_events"].join('/'), "POST", {pod_events: pod_events, node_events: node_events})
   end
 
-
-  def job_log(logs, level: 'info', step: nil)
-    if @job_type == "image"
-      @image.job_log(logs, level: level, step: step)
-    else
-      @project.job_log(logs, level: level, step: step, job_type: @job_type, job_id: @job_id)
-    end
+  def get_pod_events(pod_name)
+    return if pod_name.blank?
+    `kubectl get event --namespace cnvrg --field-selector involvedObject.name=#{pod_name} -o json`
   end
 
-
-
-  def monitor_single_command(cmd, command_slug)
-    log = ""
-    PTY.spawn(cmd[:command]) do |stdout, stdin, pid, stderr|
-      begin
-        stdout.each do |line|
-          log += line + "\n"
-          if cmd[:success_log] and line.match(cmd[:success_log])
-            puts "Match found!"
-            context = {log: line, pid: Process.pid}
-            update_command(CommandsStatus::SUCCESS, context, command_slug)
-          end
-        end
-
-        if stderr
-          stderr.each do |err|
-            log += err + "\n"
-            #context = {error: err}
-            #update_command(CommandsStatus::FAILED, context, command_slug)
-            #log << {time: Time.now, message: err, type: "stderr"}
-          end
-        end
-
-      rescue Errno::EIO => e
-        Cnvrg::Logger.log_error(e)
-        context = {error: e.message}
-        update_command(CommandsStatus::FAILED, context, command_slug)
-      rescue Errno::ENOENT => e
-        exp_success = false
-        context = {error: e.message}
-        update_command(CommandsStatus::FAILED, context, command_slug)
-        Cnvrg::Logger.info("command \"#{cmd[:command]}\" couldn't be executed, verify command is valid")
-        Cnvrg::Logger.log_error(e)
-      rescue => e
-        #res = @exp.end(log, 1, start_commit, 0, 0)
-        context = {error: e.message}
-        update_command(CommandsStatus::FAILED, context, command_slug)
-        Cnvrg::Logger.info("Error occurred,aborting")
-        Cnvrg::Logger.log_error(e)
-        exit(0)
-      end
-      ::Process.wait pid
-    end
+  def get_node_events(node_name)
+    return if node_name.blank?
+    `kubectl get event --all-namespaces --field-selector involvedObject.name=#{node_name} -o json`
   end
-
 end