cnvrg 1.6.0.5 → 1.6.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 526b6f5148f32d44e338cfbd81d7a93a79403ba835a462600ed09778c0af6180
-  data.tar.gz: 42da99517eeee50cfd3c88c50e2436d088a8526710084a11f642bf46b74acbfa
+  metadata.gz: eec6f04b04951d9939b8a6b139fb4566d524b68ca01ace06e7f2be48fa92b6bb
+  data.tar.gz: 1aa365dca2cae40afb8dc1b8bddfd0029e9f376c708c8bed2ba66741094a2016
 SHA512:
-  metadata.gz: '086eaf1433c9ce2b0ff480bd8c92fd7e3a9ba3a70cf937c9c1f5e6f749382c0ad7073a0e99f62e0efafbce92a74a1f48afa1db18a9b0f2b84423fb753571a9bc'
-  data.tar.gz: 3508f1591a67ba0d9f5122a0a667295f18a54c9f89b83e26a5ab5956e2abf369f2a0f2ceaed893b595a7809ebc0c60744231e07c83249323664c54a0c7826c2a
+  metadata.gz: 6fb5ccb7658268ca0ccfcad00a1e490468f91bf77de5313b61aead3885fe64c4c2f50248a89a24488cf016f8cc53fa53a98a5d3f891deb6740dfecd4c682adec
+  data.tar.gz: 787f8aa03cfc15989ba97ee583144e83070aa31d774886269af1910d15a2bd81338a2ae4e1a44b60a1a8b60de5b2db9cc995df0cda1f12f046af9e7fefa67283

data/lib/cnvrg/api.rb

@@ -54,7 +54,7 @@ module Cnvrg
             conn.headers['Auth-Token'] = @pass
             conn.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
             conn.options.timeout = 420
-                conn.options.open_timeout=600
+                conn.options.open_timeout=180
             case method
               when 'GET'
                 retries = 0

data/lib/cnvrg/cli.rb

@@ -865,6 +865,7 @@ module Cnvrg
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
+        return if check_pod_restart[1] ## It means that all datasets downloaded successfully
         commit = options["commit"] || commit
         only_tree = options["only_tree"] || only_tree
         read = options["read"] || read || false
@@ -1800,6 +1801,7 @@ module Cnvrg
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
+        return if check_pod_restart[0] ## It means that project downloaded successfully
         url_parts = project_url.split("/")
         project_index = Cnvrg::Helpers.look_for_in_path(project_url, "projects")
         slug = url_parts[project_index + 1]
@@ -2449,6 +2451,7 @@ module Cnvrg
         @dataset = Dataset.new(dataset_dir)
         @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
         all_files = all_files
+        @dataset.generate_idx
         res = @dataset.compare_idx_download(all_files: all_files, desired_commit: commit)
         unless CLI.is_response_success(res, false)
           log_message("Cant find the desired commit, please check it or try to download without it.", Thor::Shell::Color::RED)
@@ -5367,6 +5370,15 @@ module Cnvrg
       @image.update_image_activity(project.last_local_commit, nil)
     end
 
+    desc 'check_pod_restart', description: 'Check pod restart', hide: true
+    def check_pod_restart
+      Cnvrg::CLI.new.log_start(__method__, args, options)
+      @project = Project.new(owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
+      @project.check_job_pod_restart
+    rescue => e
+      Cnvrg::Logger.log_error(e)
+      [false, false]
+    end
 
     no_tasks do
       def get_instance_type(instances)
@@ -6161,7 +6173,6 @@ module Cnvrg
     end
   end
 
-
 end
 
 

data/lib/cnvrg/helpers/executer.rb

@@ -1,4 +1,10 @@
 class Cnvrg::Helpers::Executer
+  module CommandsStatus
+    ACK = :ack
+    STARTED = :started
+    FAILED = :failed
+    SUCCESS = :success
+  end
   def initialize(project: nil, job_type: nil, job_id: nil, image: nil)
     @image = image
     @project = project || Cnvrg::Project.new(owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
@@ -53,6 +59,10 @@ class Cnvrg::Helpers::Executer
     # :title => command title, can replace the on_error, on_success fields
     # :on_error_log => log to be logged on exit_code != 0
     # :on_success_log => log to be logged on exit_code == 0
+    # when type == 'notify_command'
+    # notify to commands api about command progress
+    # when type == 'spawn'
+    # run in another process and detach from it
     #
     retries = cmd[:retries] || 1
     resp = []
@@ -82,6 +92,22 @@ class Cnvrg::Helpers::Executer
     commands.map{|k| k.with_indifferent_access}
   end
 
+  def get_commands
+    resp = Cnvrg::API.request("#{base_url}/commands", "GET")
+    commands = resp["commands"]
+    commands.map{|k| k.with_indifferent_access}
+  end
+
+  def get_command(command_slug)
+    resp = Cnvrg::API.request("#{base_url}/commands/#{command_slug}", "GET")
+    command = resp["command"]
+    command.with_indifferent_access
+  end
+
+  def update_command(status, context, command_slug)
+    Cnvrg::API.request("#{base_url}/commands/#{command_slug}", "PUT", {status: status, context: context, timestamp: Time.now})
+  end
+
 
   def update_git_commit
     git_commit = `git rev-parse --verify HEAD`
@@ -93,11 +119,19 @@ class Cnvrg::Helpers::Executer
     Cnvrg::API.request("#{base_url}/datasets/#{dataset}", "PUT", {status: status})
   end
 
+  def monitor_command(command, command_slug)
+    monitor_single_command(command, command_slug)
+  end
+
   private
   def execute_helper(cmd)
     case cmd[:type]
     when 'notify'
       return run_and_notify(cmd)
+    when 'notify_command'
+      return notify_command(cmd)
+    when 'spawn'
+      return spawn_command(cmd)
     when 'file_exists'
       if File.exists? cmd[:file]
         return execute_cmds(cmd[:exists_commands]) if cmd[:exists_commands].present?
@@ -111,6 +145,22 @@ class Cnvrg::Helpers::Executer
     end
   end
 
+  def notify_command(cmd)
+    exit_status, output, errors, start_time, end_time = regular_command(cmd)
+    context ={
+        exit_status: exit_status,
+        output: output,
+        errors: errors,
+        start_time: start_time,
+        end_time: end_time
+    }
+    if exit_status == 0
+      update_command(CommandsStatus::SUCCESS, context, cmd[:command_slug])
+    else
+      update_command(CommandsStatus::FAILED, context, cmd[:command_slug])
+    end
+  end
+
   def run_and_notify(cmd)
     with_logs = cmd[:logs]
     cmd = init_cmd_logs(cmd)
@@ -134,6 +184,11 @@ class Cnvrg::Helpers::Executer
   end
 
 
+  def spawn_command(cmd={})
+    pid = Process.spawn(cmd[:command])
+    Process.detach(pid)
+  end
+
   def regular_command(cmd = {})
     errors = []
     output = []
@@ -205,4 +260,50 @@ class Cnvrg::Helpers::Executer
     end
   end
 
+
+
+  def monitor_single_command(cmd, command_slug)
+    log = ""
+    PTY.spawn(cmd[:command]) do |stdout, stdin, pid, stderr|
+      begin
+        stdout.each do |line|
+          log += line + "\n"
+          if cmd[:success_log] and line.match(cmd[:success_log])
+            puts "Match found!"
+            context = {log: line, pid: Process.pid}
+            update_command(CommandsStatus::SUCCESS, context, command_slug)
+          end
+        end
+
+        if stderr
+          stderr.each do |err|
+            log += err + "\n"
+            #context = {error: err}
+            #update_command(CommandsStatus::FAILED, context, command_slug)
+            #log << {time: Time.now, message: err, type: "stderr"}
+          end
+        end
+
+      rescue Errno::EIO => e
+        Cnvrg::Logger.log_error(e)
+        context = {error: e.message}
+        update_command(CommandsStatus::FAILED, context, command_slug)
+      rescue Errno::ENOENT => e
+        exp_success = false
+        context = {error: e.message}
+        update_command(CommandsStatus::FAILED, context, command_slug)
+        Cnvrg::Logger.info("command \"#{cmd[:command]}\" couldn't be executed, verify command is valid")
+        Cnvrg::Logger.log_error(e)
+      rescue => e
+        #res = @exp.end(log, 1, start_commit, 0, 0)
+        context = {error: e.message}
+        update_command(CommandsStatus::FAILED, context, command_slug)
+        Cnvrg::Logger.info("Error occurred,aborting")
+        Cnvrg::Logger.log_error(e)
+        exit(0)
+      end
+      ::Process.wait pid
+    end
+  end
+
 end

data/lib/cnvrg/job_cli.rb

@@ -4,10 +4,15 @@ module Cnvrg
         desc 'log', description: '', hide: true
         method_option :level, :type => :string, :aliases => ["-l", "--level"], :default => 'info'
         method_option :step, :type => :string, :aliases => ["-s", "--step"], :default => nil
+        method_option :restart, :type => :boolean, :aliases => ["-r", "--restart"], :default => false
         def log(*logs)
             Cnvrg::CLI.new.log_start(__method__, args, options)
             @project = Project.new(owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
-            @project.job_log(logs, level: options['level'], step: options['step'])
+            if options['restart'] == @project.check_job_pod_restart[0] or options['step'] == "ready"
+                @project.job_log(logs, level: options['level'], step: options['step'])
+            else
+                @project.job_log(nil, level: options['level'], step: options['step'])
+            end
         end
 
 
@@ -38,19 +43,57 @@ module Cnvrg
             commands = @executer.get_requirements_commands
             @executer.execute_cmds(commands)
         end
-        desc 'start/stop tensorboard',  hide: true
-        def tensorboard(start:true)
-          cli = Cnvrg::CLI.new
-          cli.log_start(__method__, args, options)
-          @project = Project.new(nil, owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
-          @executer = Helpers::Executer.new(project: @project, job_type: ENV['CNVRG_JOB_TYPE'], job_id: ENV['CNVRG_JOB_ID'])
-          commands = []
-          if start
-            commands << "nohup tensorboard --logdir=/home/ds/notebooks </dev/null &"
-            @executer.execute_cmds(commands)
-          else
 
-          end
+        desc 'poll_commands', description: 'Poll waiting commands', hide: true
+        def poll_commands
+            cli = Cnvrg::CLI.new
+            cli.log_start(__method__, args, options)
+            @project = Project.new(nil, owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
+            @executer = Helpers::Executer.new(project: @project, job_type: ENV['CNVRG_JOB_TYPE'], job_id: ENV['CNVRG_JOB_ID'])
+            while true
+                begin
+                    commands = @executer.get_commands
+                    @executer.execute_cmds(commands)
+                rescue => e
+                    Cnvrg::Logger.log_error(e)
+                end
+                sleep 10
+            end
+        end
+
+        desc 'run_command', description: 'Poll waiting commands', hide: true
+        method_option :command_slug, :type => :string, :aliases => ["-s", "--slug"], :default => nil
+        def run_command
+            Cnvrg::Logger.info("in run command")
+            cli = Cnvrg::CLI.new
+            cli.log_start(__method__, args, options)
+            command_slug = options['command_slug']
+            #TODO handle if missing
+            @project = Project.new(nil, owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
+            @executer = Helpers::Executer.new(project: @project, job_type: ENV['CNVRG_JOB_TYPE'], job_id: ENV['CNVRG_JOB_ID'])
+            command = @executer.get_command(command_slug)
+            @executer.monitor_command(command, command_slug)
+        end
+
+        desc 'set_pod_restart', description: 'Set pod restart', hide: true
+        def set_pod_restart
+            Cnvrg::CLI.new.log_start(__method__, args, options)
+            @project = Project.new(owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
+            @project.set_job_pod_restart
+        end
+
+        desc 'pre_pod_restart', description: 'Pre pod restart actions', hide: true
+        def pre_pod_restart
+            Cnvrg::CLI.new.log_start(__method__, args, options)
+            @project = Project.new(owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
+            @project.pre_job_pod_restart
+        end
+
+        desc 'started', description: 'set job status to started after job restart', hide: true
+        def started
+            Cnvrg::CLI.new.log_start(__method__, args, options)
+            @project = Project.new(owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
+            @project.set_job_started
         end
     end
 end

data/lib/cnvrg/project.rb

@@ -649,7 +649,9 @@ module Cnvrg
 
 
     def update_job_jupyter_token(job_type, job_id, token)
-      base_url = "users/#{self.owner}/projects/#{self.slug}/jobs/#{job_type.underscore}/#{job_id}"
+      owner = self.owner || ENV['CNVRG_OWNER']
+      slug = self.slug || ENV['CNVRG_PROJECT']
+      base_url = "users/#{owner}/projects/#{slug}/jobs/#{job_type.underscore}/#{job_id}"
       Cnvrg::API.request("#{base_url}/update_jupyter_token", "POST", {token: token})
     end
 
@@ -736,6 +738,43 @@ module Cnvrg
       end
     end
 
+    def set_job_pod_restart(job_type: nil, job_id: nil)
+      job_type ||= ENV['CNVRG_JOB_TYPE']
+      job_id ||= ENV['CNVRG_JOB_ID']
+      if job_type.blank? or job_id.blank?
+        raise StandardError.new("Cant find job env variables")
+      end
+      Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/set_pod_restart", "POST", {job_type: job_type, job_id: job_id})
+    end
+
+    def check_job_pod_restart(job_type: nil, job_id: nil)
+      job_type ||= ENV['CNVRG_JOB_TYPE']
+      job_id ||= ENV['CNVRG_JOB_ID']
+      if job_type.blank? or job_id.blank?
+        raise StandardError.new("Cant find job env variables")
+      end
+      resp = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/check_pod_restart", "GET", {job_type: job_type, job_id: job_id})
+      return [false, false] if resp.blank?
+      Cnvrg::Logger.log_info("Checked for pod restart got response #{resp}")
+      [resp['project_downloaded'], resp['dataset_downloaded']]
+    end
 
+    def pre_job_pod_restart
+      job_type ||= ENV['CNVRG_JOB_TYPE']
+      job_id ||= ENV['CNVRG_JOB_ID']
+      if job_type.blank? or job_id.blank?
+        raise StandardError.new("Cant find job env variables")
+      end
+      Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/pre_pod_restart", "POST", {job_type: job_type, job_id: job_id})
+    end
+
+    def set_job_started
+      job_type ||= ENV['CNVRG_JOB_TYPE']
+      job_id ||= ENV['CNVRG_JOB_ID']
+      if job_type.blank? or job_id.blank?
+        raise StandardError.new("Cant find job env variables")
+      end
+      Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/set_started", "POST", {job_type: job_type, job_id: job_id})
+    end
   end
 end

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '1.6.0.5'
+  VERSION = '1.6.0.6'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.6.0.5
+  version: 1.6.0.6
 platform: ruby
 authors:
 - Yochay Ettun
@@ -476,7 +476,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.4
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.