cnvrg 1.9.9.9.7

data/lib/cnvrg/helpers/agent.rb

@@ -0,0 +1,188 @@
+class Cnvrg::Helpers::Agent
+
+  module Status
+    STARTED = :started
+    RUNNING = :running
+    FINISHED = :finished
+    ABORTED = "aborted"
+  end
+
+  module LogLevel
+    INFO = :info
+    PURE = :pure
+    ERROR = :error
+  end
+
+  #### This class represent a single command in the system.
+  #### it runs under an executer (machine_activity) so it should have all the executer
+  #### params
+  def initialize(executer: nil, slug: nil, command: nil, container_name: nil, send_log_interval: 60, timeout: -1, logs_regex: [], async: false, send_logs: false, files_exist: [], retries: 0, sleep_before_retry: 30, single_quotes: false, docker_user: nil, use_bash: false, **kwargs)
+    @executer = executer
+    @slug = slug
+    @files_exist = files_exist
+    @container_name = container_name
+    @run_in_slave = @container_name.downcase == "slave"
+    @log_interval = send_log_interval
+    # https://ruby-doc.org/stdlib-2.5.1/libdoc/timeout/rdoc/Timeout.html timeout should be 0 for running forever
+    if timeout.blank? or timeout.negative?
+      @timeout = 0
+    else
+      @timeout = timeout
+    end
+    @logs_regex = logs_regex || []
+    @async = async
+    @command = command
+    @send_logs = send_logs
+    @retries = retries.try(:to_i) ## How many times the user asked to try to execute the command again
+    @sleep_before_retry = sleep_before_retry
+    @real_execution_retries = 0 ## How many times the command really executed until success
+    @single_quotes = single_quotes
+    @docker_user = ""
+    @shell_type = use_bash ? "bash -l" : "sh"
+    if docker_user.present?
+      @docker_user = " --user #{docker_user}"
+    end
+    if @run_in_slave
+      if @single_quotes
+        @command = "docker exec #{@docker_user} -it #{@executer.slave_id} #{@shell_type} -c '#{@command}'"
+      else
+        @command = "docker exec #{@docker_user} -it #{@executer.slave_id} #{@shell_type} -c \"#{@command}\""
+      end
+    end
+    @output = []
+    @errors = []
+    @exit_status = nil
+    @is_running = true
+    @pid = nil
+  end
+
+  def base_url
+    [@executer.activity_url, "commands", @slug].join("/")
+  end
+
+  def should_run?
+    if @files_exist.present?
+      file_doesnt_exists = @files_exist.find do |file|
+        not File.exists? file
+      end
+      return true if file_doesnt_exists.blank?
+      log_internal("Can't find file #{file_doesnt_exists}, stopping the job")
+      return false
+    end
+    true
+  end
+
+
+  def exec!
+    log_internal("Command: #{@command} with slug: #{@slug} started!")
+    if should_run?
+      send_logs(status: Status::STARTED)
+      periodic_thread
+      execute_command
+    else
+      @exit_status = 127
+    end
+    finish_log = "Command: #{@command} with slug: #{@slug} finished"
+    finish_log += " after #{@real_execution_retries} retries" if @real_execution_retries > 0
+    log_internal(finish_log)
+    send_logs(exit_status: @exit_status, status: Status::FINISHED)
+  end
+
+  def get_logs_to_send
+    new_logs = @output.pop(@output.length)
+    new_errors = @errors.pop(@errors.length)
+    [new_logs, new_errors]
+  end
+
+
+  def periodic_thread
+    Thread.new do
+        while @exit_status.blank?
+          Thread.exit if @log_interval.blank?
+          sleep(@log_interval)
+          send_logs
+        end
+    end
+  end
+
+  def retry_command
+    @retries -=1
+    sleep @sleep_before_retry
+    @real_execution_retries +=1
+    execute_command
+  end
+
+  def execute_command
+    Timeout.timeout(@timeout) do
+      PTY.spawn(@command) do |stdout, stdin, pid, stderr|
+        @pid = pid
+        begin
+          if stdout.present?
+            stdout.each do |line|
+              log_internal(line, level: LogLevel::PURE)
+              line = line.strip.gsub(/\e\[([;\d]+)?m/, '')
+              @output << {log: line, timestamp: Time.now}
+            end
+          end
+
+          if stderr.present?
+            stderr.each do |line|
+              line = line.strip.gsub(/\e\[([;\d]+)?m/, '')
+              log_internal(line, level: LogLevel::ERROR)
+              @errors << {log: line, timestamp: Time.now}
+            end
+          end
+        rescue Errno::EIO => e
+          next
+        rescue => e
+          log_internal(e.message, level: LogLevel::ERROR)
+          log_internal(e.backtrace.join("\n"), level: LogLevel::ERROR)
+          @errors << {log: e.message, timestamp: Time.now}
+        end
+        ::Process.wait pid
+      end
+    end
+    @exit_status = $?.exitstatus
+  rescue Timeout::Error
+    Process.kill(0, @pid)
+    @errors << {log: "Command timed out!", timestamp: Time.now}
+    log_internal("Command timed out!", level: LogLevel::ERROR)
+    @exit_status = 124
+  ensure
+    retry_command if @retries != 0 and @exit_status !=0
+    @exit_status
+  end
+
+  private
+  def send_logs(exit_status: nil, status: Status::RUNNING)
+    logs, error_logs = get_logs_to_send
+    # Filter logs only if not failed
+    if exit_status.blank? or exit_status == 0
+      logs = filter_logs_by_regex(logs)
+    end
+    ### there is no logs, no exit_status and status is running.
+    ### this condition let us call "send_logs" every interval iteration.
+    if logs.blank? and error_logs.blank? and exit_status.blank? and status == Status::RUNNING
+      return
+    end
+    Cnvrg::API.request(base_url, 'PUT', {logs: logs, error_logs: error_logs, exit_status: exit_status, status: status, execution_retries: @real_execution_retries, pid: @pid})
+  end
+
+  def log_internal(log, level: LogLevel::INFO)
+    if level == LogLevel::PURE
+      puts(log)
+    else
+      puts({log: log, level: level, timestamp: Time.now, command: @slug, machine_activity: @executer.machine_activity}.to_json)
+    end
+    STDOUT.flush
+  end
+
+  def filter_logs_by_regex(logs)
+    logs.select do |log|
+      next true if @send_logs
+      @logs_regex.find do |regexp_str|
+        Regexp.new(regexp_str).match(log[:log]).present?
+      end
+    end
+  end
+end

data/lib/cnvrg/helpers/executer.rb

@@ -0,0 +1,213 @@
+require 'cnvrg/helpers/agent'
+class Cnvrg::Helpers::Executer
+  attr_reader :machine_activity, :agent_id, :slave_id
+
+
+  ### this class represent a machine_activity. it will poll the commands, communicate with the
+  # server (poll commands) and let the server know the status of this executer.
+  def initialize(owner: nil, machine_activity: nil, poll_every: 30, job_id: nil)
+    @owner = owner
+    @job_id = job_id
+    @poll_every = poll_every
+    @machine_activity = machine_activity
+    @commands_q = Queue.new
+    @files_q = Queue.new
+    @agent_id = nil
+    @slave_id = nil
+  end
+
+  def create_file_cmd(path, content)
+    if path.include? "~"
+      path = File.expand_path(path)
+    end
+    FileUtils.mkdir_p(File.dirname(path))
+    File.open(path, "w+"){|f| f.write(content)}
+  end
+
+  def handle_files(files)
+    (files || {}).each do |path, content|
+      create_file_cmd(path, content)
+    end
+  end
+
+  def activity_url
+    ['users', @owner, 'machine_activities', @machine_activity].join("/")
+  end
+
+  def executer_stats
+    return @stats if @stats.present?
+    Cnvrg::Logger.log_info("getting containers")
+    @agent_id, @slave_id = containers
+    Cnvrg::Logger.log_info("got containers")
+    pod_name, node_name = get_node_and_pod_names
+    @stats = {
+        pod_name: pod_name,
+        node_name: node_name,
+        agent: {
+          container_id: @agent_id,
+          workdir: `pwd`.strip,
+          homedir: current_homedir,
+          user: `whoami`.strip,
+          user_id: `id -u`.strip,
+          group_id: `id -g`.strip,
+          cnvrg: Cnvrg::VERSION
+        },
+        slave: {
+            container_id: @slave_id,
+            workdir: run_in_slave('pwd'),
+            homedir: slave_homedir,
+            spark_path: spark_path,
+            user: run_in_slave( 'whoami'),
+            cnvrg: run_in_slave( 'which cnvrg'),
+            has_bash: run_in_slave( 'which bash'),
+            user_id: run_in_slave( 'id -u'),
+            group_id: run_in_slave( 'id -g'),
+            python_version: run_in_slave( 'python --version'),
+            python3_version: run_in_slave( 'python3 --version'),
+            pip_version: run_in_slave( 'pip --version'),
+            pip3_version: run_in_slave( 'pip3 --version')
+        },
+    }
+    @stats
+  end
+
+  def containers
+    agent_id = nil
+    slave_id = nil
+    while agent_id.blank? or slave_id.blank?
+      grep_by = @job_id
+      grep_by = "$(hostname)" if ENV['KUBERNETES_PORT'].present?
+      cntrs = `docker ps --format "table {{.ID}},{{.Names}}" | grep -i #{grep_by}`.split("\n").map{|x| x.strip}
+      agent_id = cntrs.find{|container_name| container_name.include? "agent"}.split(",").first rescue nil
+      slave_id = cntrs.find{|container_name| container_name.include? "slave"}.split(",").first rescue nil
+      sleep(5)
+    end
+    if slave_id.blank?
+      raise "Can't find slave id"
+    end
+    [agent_id, slave_id]
+  end
+
+  def current_homedir
+    `env | grep -w HOME`.strip.split("=").try(:last)
+  end
+
+  def spark_path
+    run_in_slave("env | grep SPARK_HOME").strip.split("=").try(:last)
+  end
+
+  def slave_homedir()
+    run_in_slave("env | grep -w HOME").split("=").try(:last)
+  end
+
+  def slave_env
+    run_in_slave("env").split("\n").map{|x| x.split("=")}
+  end
+
+  def run_in_slave(command)
+    `docker exec -i #{@slave_id} sh -c '#{command}'`.strip
+  end
+
+
+  def poll
+    resp = Cnvrg::API.request([activity_url, "commands"].join('/'), "POST")
+    commands = resp["commands"]
+    files = resp["files"]
+    handle_files(files)
+    commands.each{|cmd| @commands_q.push(cmd)}
+  rescue => e
+    Cnvrg::Logger.log_error(e)
+  end
+
+  def init
+    resp = Cnvrg::API.request(activity_url, "PUT", {stats: executer_stats})
+    machine_activity = resp["machine_activity"]
+    Cnvrg::Logger.log_info("Got back machine activity #{machine_activity}")
+    if machine_activity.present? and @machine_activity != machine_activity
+      Cnvrg::Logger.log_info("Changing to machine activity #{machine_activity}")
+      machine_activity_yml = {slug: machine_activity}
+      File.open("/conf/.machine_activity.yml", "w+") {|f| f.write machine_activity_yml.to_yaml}
+      @machine_activity = machine_activity
+    end
+  rescue => e
+    Cnvrg::Logger.log_error(e)
+  end
+
+  def polling_thread
+    while true
+      poll
+      sleep(@poll_every)
+    end
+  end
+
+  def main_thread
+    init
+    Thread.new do
+      polling_thread
+    end
+    execute_cmds
+  end
+
+  def execute_cmds
+    pids = []
+    while true
+      if @commands_q.empty?
+        sleep(5)
+        next
+      end
+      cmd = @commands_q.pop.symbolize_keys
+      command_json = Cnvrg::API.request([activity_url, "commands", cmd[:slug]].join('/'), "GET")
+
+      cmd_status = command_json["status"] rescue ""
+
+      if cmd_status == Cnvrg::Helpers::Agent::Status::ABORTED
+        Cnvrg::Logger.log_info("stopping job because command #{cmd[:slug]} with status #{cmd_status}")
+        next
+      end
+      pid = Process.fork do
+        Cnvrg::Helpers::Agent.new(executer: self, **cmd).exec!
+      end
+      if cmd[:async].blank?
+        Process.waitpid(pid)
+      else
+        Process.detach(pid)
+      end
+      pids << pid
+      ######
+    end
+    pids
+  end
+
+  def merge_log_block(logs)
+    logs.group_by {|log| log[:timestamp].to_s}
+        .map {|ts, logz| {timestamp: ts, logs: logz.map {|l| l[:log]}.join("\n")}}
+  end
+
+  def get_node_and_pod_names
+    pod_name = `hostname`.strip rescue nil
+    node_name = nil
+    if pod_name.present?
+      pod_describe = `kubectl -n cnvrg get pod #{pod_name} -o json` rescue nil
+      pod_describe = JSON.parse(pod_describe) rescue {}
+      node_name = pod_describe["spec"]["nodeName"] rescue nil
+    end
+    [pod_name, node_name]
+  end
+
+  def pre_pod_stop
+    pod_name, node_name = get_node_and_pod_names
+    pod_events = get_pod_events(pod_name)
+    node_events = get_node_events(node_name)
+    Cnvrg::API.request([activity_url, "job_events"].join('/'), "POST", {pod_events: pod_events, node_events: node_events})
+  end
+
+  def get_pod_events(pod_name)
+    return if pod_name.blank?
+    `kubectl get event --namespace cnvrg --field-selector involvedObject.name=#{pod_name} -o json`
+  end
+
+  def get_node_events(node_name)
+    return if node_name.blank?
+    `kubectl get event --all-namespaces --field-selector involvedObject.name=#{node_name} -o json`
+  end
+end

data/lib/cnvrg/hyper.rb

@@ -0,0 +1,21 @@
+module Cnvrg
+  class Hyper
+    def initialize(project_path, path)
+      @project = Cnvrg::Project.new(project_path)
+      @content = YAML.load_file(path)
+      @base_resource = "users/#{@project.owner}"
+      @params = []
+    end
+
+    def resolve_params
+      resp = Cnvrg::API.request(@base_resource + "/resolve_grid", "POST", {hyper_search: @content})
+      unless Cnvrg::CLI.is_response_success(resp, false)
+        return nil
+      end
+      resp['result']['params'].each do |param|
+        @params << {key: param.first.keys.first, value: param.map{|p| p.values}.flatten.join(',')}
+      end
+      @params
+    end
+  end
+end

data/lib/cnvrg/image.rb

@@ -0,0 +1,113 @@
+module Cnvrg
+  class Image
+    # attr_reader :image_name, :image_tag, :is_docker, :project_slug, :commit_id, :owner, :port, :image_slug
+
+
+    def initialize(image_id)
+      begin
+        @cli = Cnvrg::CLI.new
+        home_dir = File.expand_path('~')
+        config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+        @owner = config.to_h[:owner]
+        @username =  config.to_h[:username]
+        @image_id = image_id
+      rescue => e
+        @owner = ""
+        @username =  ""
+        @cli.log_message("cnvrg is not configured")
+      end
+    end
+
+    def build
+      image_data = get_image_data
+      file_name = "Dockerfile-#{@image_id}"
+      File.new(file_name, "w+")
+      if image_data["reqs_file_path"].present? and image_data["from_image_name"]
+        File.open(file_name, "w+") do |i|
+          i.write("FROM #{image_data["from_image_name"]}")
+          i.write("ADD requirements.txt requirements.txt")
+          i.write("RUN pip3 install -r requirements.txt")
+        end
+      else
+        open(file_name, 'wb') do |file|
+          file << open(image_data["docker_file_path"]).read
+        end
+      end
+        docker_url = "#{image_data["docker_name"]}"
+        command = {:type=>"notify",
+                   :title=>"docker build",
+                   :logs=>true,
+                  :before_execute_log=>"Building docker image",
+                  :timeout=>3600,
+                   :command=>"sudo docker build . -t #{docker_url} -f #{file_name}"}
+      @executer = Helpers::Executer.new(project: @project, job_type: "image", job_id: @image_id, image: self)
+      exit_status, output, errors, _, _ = @executer.execute(command)
+      all_logs = join_logs(output, errors)
+      if exit_status != 0
+        raise StandardError.new(all_logs)
+      end
+      if ENV["CNVRG_IMAGE_BUILD_USERNAME"].present? and ENV["CNVRG_IMAGE_BUILD_PASSWORD"].present?
+        if ENV["CNVRG_IMAGE_BUILD_REGISTRY"].present?
+        command = {:type=>"notify",
+                    :no_stdout => true,
+                    :title=>"docker login",
+                    :logs=>true,
+                    :command=>"sudo docker login #{ENV["CNVRG_IMAGE_BUILD_REGISTRY"]} --username=#{ENV["CNVRG_IMAGE_BUILD_USERNAME"]} --password=\"#{ENV["CNVRG_IMAGE_BUILD_PASSWORD"]}\""}
+        else
+          command = {:type=>"notify",
+                     :no_stdout => true,
+                     :title=>"docker login",
+                     :logs=>true,
+                     :command=>"sudo docker login --username=#{ENV["CNVRG_IMAGE_BUILD_USERNAME"]} --password=\"#{ENV["CNVRG_IMAGE_BUILD_PASSWORD"]}\""}
+          end
+        exit_status, output, errors, _, _ = @executer.execute(command)
+        all_logs = join_logs(output, errors)
+        if exit_status != 0
+          raise StandardError.new(all_logs)
+        end
+      end
+      command = {:type=>"notify",
+                  :title=>"docker push",
+                  :logs=>true,
+                 :before_execute_log=>"Pushing docker image",
+                 :timeout=>3600,
+                 :command=>"sudo docker push #{docker_url}"}
+      exit_status, output, errors, _, _ = @executer.execute(command)
+      all_logs = join_logs(output, errors)
+      if exit_status != 0
+        raise StandardError.new(all_logs)
+      end
+      post_build_update(true)
+    rescue => e
+      @cli.log_message("Image Build failed")
+      post_build_update(false, e.message)
+    end
+
+    def get_image_data
+      response = Cnvrg::API.request("users/#{@owner}/images/#{@image_id}/image_start_build", 'GET')
+      CLI.is_response_success(response)
+      return response["image"]
+    end
+
+    def post_build_update(success, message = "")
+      response = Cnvrg::API.request("users/#{@owner}/images/#{@image_id}/image_end_build", 'POST', {success: success, message: message})
+      CLI.is_response_success(response)
+      return response["image"]
+    end
+
+
+    def job_log(logs, level: 'info', step: nil, job_type: "image", job_id: @image_id)
+      logs = [logs].flatten
+      logs.each_slice(10).each do |temp_logs|
+        Cnvrg::API.request("users/#{@owner}/images/#{@image_id}/log", "POST", {job_type: job_type, job_id: job_id, logs: temp_logs, log_level: level, step: step, timestamp: Time.now})
+        sleep(1)
+      end
+    end
+
+
+    def join_logs(output, errors)
+      output.map{ |o| o[:logs]}.join(" ") + " " + errors.map{ |o| o[:logs]}.join(" ")
+    end
+
+  end
+end