cnvrg 1.6.38 → 1.9.5

data/lib/cnvrg/job_cli.rb

@@ -15,17 +15,6 @@ module Cnvrg
             end
         end
 
-
-        desc 'start', description: '', hide: true
-        def start(*logs)
-            cli = Cnvrg::CLI.new
-            cli.log_start(__method__, args, options)
-            @project = Project.new(nil, owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
-            @executer = Helpers::Executer.new(project: @project, job_type: ENV['CNVRG_JOB_TYPE'], job_id: ENV['CNVRG_JOB_ID'])
-            commands = @executer.fetch_commands
-            @executer.execute_cmds(commands)
-        end
-
         desc 'requirements', description: '', hide: true
         def requirements
           cli = Cnvrg::CLI.new
@@ -44,56 +33,42 @@ module Cnvrg
             @executer.execute_cmds(commands)
         end
 
-        desc 'poll_commands', description: 'Poll waiting commands', hide: true
-        def poll_commands
+
+        desc 'start', description: "Job Start!", hide: true
+        def start
             cli = Cnvrg::CLI.new
             cli.log_start(__method__, args, options)
-            @project = Project.new(nil, owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
-            @executer = Helpers::Executer.new(project: @project, job_type: ENV['CNVRG_JOB_TYPE'], job_id: ENV['CNVRG_JOB_ID'])
-            while true
-                begin
-                    commands = @executer.get_commands
-                    @executer.execute_cmds(commands)
-                rescue => e
-                    Cnvrg::Logger.log_error(e)
-                end
-                sleep 10
-            end
+            cli.auth
+            poll_every = (ENV["CNVRG_AGENT_POLL_EVERY"] || '30').to_i
+            owner = ENV["CNVRG_OWNER"]
+            machine_activity_slug = ENV["CNVRG_MACHINE_ACTIVITY"]
+            job_id = ENV["CNVRG_JOB_ID"]
+            @executer = Cnvrg::Helpers::Executer.new(machine_activity: machine_activity_slug, poll_every: poll_every, owner: owner, job_id: job_id)
+            @executer.main_thread
         end
 
-        desc 'run_command', description: 'Poll waiting commands', hide: true
-        method_option :command_slug, :type => :string, :aliases => ["-s", "--slug"], :default => nil
-        def run_command
-            Cnvrg::Logger.info("in run command")
+        desc 'stats', description: 'stats of agent and slave', hide: true
+        def stats
             cli = Cnvrg::CLI.new
             cli.log_start(__method__, args, options)
-            command_slug = options['command_slug']
-            #TODO handle if missing
-            @project = Project.new(nil, owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
-            @executer = Helpers::Executer.new(project: @project, job_type: ENV['CNVRG_JOB_TYPE'], job_id: ENV['CNVRG_JOB_ID'])
-            command = @executer.get_command(command_slug)
-            @executer.monitor_command(command, command_slug)
+            poll_every = (ENV["CNVRG_AGENT_POLL_EVERY"] || '30').to_i
+            owner = ENV["CNVRG_OWNER"]
+            machine_activity_slug = ENV["CNVRG_MACHINE_ACTIVITY"]
+            job_id = ENV["CNVRG_JOB_ID"]
+            @executer = Cnvrg::Helpers::Executer.new(machine_activity: machine_activity_slug, poll_every: poll_every, owner: owner, job_id: job_id)
+            @executer.executer_stats
         end
 
-        desc 'set_pod_restart', description: 'Set pod restart', hide: true
-        def set_pod_restart
-            Cnvrg::CLI.new.log_start(__method__, args, options)
-            @project = Project.new(owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
-            @project.set_job_pod_restart
-        end
-
-        desc 'pre_pod_restart', description: 'Pre pod restart actions', hide: true
-        def pre_pod_restart
-            Cnvrg::CLI.new.log_start(__method__, args, options)
-            @project = Project.new(owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
-            @project.pre_job_pod_restart
-        end
-
-        desc 'started', description: 'set job status to started after job restart', hide: true
-        def started
-            Cnvrg::CLI.new.log_start(__method__, args, options)
-            @project = Project.new(owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
-            @project.set_job_started
+        desc 'pre_pod_stop', description: 'test', hide: true
+        def pre_pod_stop
+            cli = Cnvrg::CLI.new
+            cli.log_start(__method__, args, options)
+            owner = ENV["CNVRG_OWNER"]
+            machine_activity = YAML.load_file("/conf/.machine_activity.yml") rescue {:slug => ENV["CNVRG_MACHINE_ACTIVITY"]}
+            machine_activity_slug = machine_activity[:slug]
+            job_id = ENV["CNVRG_JOB_ID"]
+            @executer = Cnvrg::Helpers::Executer.new(machine_activity: machine_activity_slug, owner: owner, job_id: job_id)
+            @executer.pre_pod_stop
         end
     end
 end

data/lib/cnvrg/job_ssh.rb

@@ -0,0 +1,47 @@
+module Cnvrg
+  class JobSsh < SubCommandBase
+    desc 'ssh start', description: 'job_id', hide: false
+    method_option :port, :type => :numeric, :aliases => ["-p", "--port"], :desc => "Port to bind into", :default => 2222
+    method_option :username, :type => :string, :aliases => ["-u", "--username"], :desc => "Job container user name", :default => nil
+    method_option :password, :type => :string, :aliases => ["--password"], :desc =>"Job Conatainer user name, will be set by cnvrg", :default => nil
+    method_option :kubeconfig, :type => :string, :aliases => ["--kubeconfig"], :desc => "Path to kubeconfig, if blank default config will be used", :default => nil
+    def start(job_id)
+      Cnvrg::CLI.new.log_start(__method__, args, options)
+      @job_ssh = ConnectJobSsh.new(job_id)
+      @job_ssh.start(options['username'], options['password'])
+      pod_name = nil
+      namespace = "cnvrg"
+      ssh_ready = false
+      while not ssh_ready
+        resp = @job_ssh.status()
+        status = resp["ssh_status"]
+
+        if status == "in_progress"
+            puts("Waiting for ssh to start ...")
+            sleep(3)
+            next
+        elsif status == "finished"
+            password = resp["password"]
+            username = resp["username"]
+            pod_name = resp["pod_name"]
+            namespace = resp["namespace"]
+            ssh_ready = true
+        else
+          puts("Failed to start ssh")
+          break
+        end 
+      end
+      if pod_name.blank? or password.blank? or username.blank?
+        puts("Failed to get required params")
+        return
+      end
+
+      puts("In order to connect to your job, define your ssh connection with the following params:")
+      puts("host: 127.0.0.1")
+      puts("port: #{options["port"]}")
+      puts("username: #{username}")
+      puts("password: #{password}")
+      @job_ssh.run_portforward_command(pod_name, options["port"], options["kubeconfig"], namespace)
+    end
+  end
+end

data/lib/cnvrg/logger.rb

@@ -75,6 +75,7 @@ module Cnvrg
 
     def log_error(e)
       Cnvrg::Logger.log_handler if $log.blank?
+      return if $log.blank?
       bc = ActiveSupport::BacktraceCleaner.new
       bc.add_silencer{|line| line =~ /thor/}
       $log.error message: "An exception #{e.class} was logged during running", type: "error"
@@ -84,16 +85,19 @@ module Cnvrg
 
     def log_error_message(msg)
       Cnvrg::Logger.log_handler if $log.blank?
+      return if $log.blank?
       $log.error message: msg, type: "error"
     end
 
     def log_info(msg)
       Cnvrg::Logger.log_handler if $log.blank?
+      return if $log.blank?
       $log.info message: msg, type: "info"
     end
 
     def log_json(json, msg: '')
       Cnvrg::Logger.log_handler if $log.blank?
+      return if $log.blank?
       $log.info message: msg, type: "info", data: json
     end
 

data/lib/cnvrg/project.rb

@@ -1,7 +1,6 @@
 require 'fileutils'
 require 'pathname'
 
-
 module Cnvrg
   class Project
     attr_reader :slug, :owner, :title, :local_path, :working_dir, :is_git, :is_branch, :machines
@@ -270,18 +269,19 @@ module Cnvrg
     end
 
     def self.clone_dir_remote(project_slug, project_owner, project_name, is_git = false)
-      list_dirs = [
-          ".cnvrg"
-      ]
+      cli = Cnvrg::CLI.new()
+      begin
+        list_dirs = [
+            ".cnvrg"
+        ]
 
 
-      list_files = [
-          ".cnvrg/config.yml",
+        list_files = [
+            ".cnvrg/config.yml",
 
-      ]
+        ]
 
 
-      begin
         config = {project_name: project_name,
                   project_slug: project_slug,
                   owner: project_owner,
@@ -300,11 +300,12 @@ module Cnvrg
 
 
         end
-
-      rescue
-        return false
+        true
+      rescue => e
+        cli.log_message(e.message)
+        cli.log_error(e)
+        false
       end
-      return true
     end
 
     def update_is_new_branch(new_branch)
@@ -396,6 +397,7 @@ module Cnvrg
       Cnvrg::Logger.log_info("Uploading: #{upload_list.join(", ")}")
       upload_list
     end
+
     def generate_output_dir_tmp(output_dir)
       upload_list = []
       list = Dir.glob("#{output_dir}/**/*", File::FNM_DOTMATCH)
@@ -449,11 +451,18 @@ module Cnvrg
 
           tree_idx[label + "/"] = nil
         else
-          sha1 = OpenSSL::Digest::SHA1.file(e).hexdigest
+          file_in_idx = old_idx[:tree][label] rescue nil
+          last_modified = File.mtime(e).to_f
+          if file_in_idx.present? and last_modified == file_in_idx[:last_modified]
+            sha1 = file_in_idx[:sha1]
+          else
+            sha1 = OpenSSL::Digest::SHA1.file(e).hexdigest
+          end
+
           if old_idx.nil? or old_idx.to_h[:tree].nil?
-            tree_idx[label] = {sha1: sha1, commit_time: nil}
-          elsif old_idx[:tree][label].nil? or old_idx[:tree][label][:sha1] != sha1
-            tree_idx[label] = {sha1: sha1, commit_time: nil}
+            tree_idx[label] = {sha1: sha1, commit_time: nil, last_modified: last_modified}
+          elsif file_in_idx.nil? or file_in_idx[:sha1] != sha1 or file_in_idx[:last_modified].blank? or file_in_idx[:last_modified] != last_modified
+            tree_idx[label] = {sha1: sha1, commit_time: nil, last_modified: last_modified}
           else
             tree_idx[label] = old_idx[:tree][label]
           end
@@ -634,6 +643,13 @@ module Cnvrg
       return response
     end
 
+    def fetch_webapp_slugs(webapp_slug, slugs: nil)
+      response = Cnvrg::API_V2.request("#{self.owner}/projects/#{self.slug}/webapps/#{webapp_slug}" , 'GET')
+      return response["experiments"]
+    rescue
+      slugs
+    end
+
     def update_idx_with_commit!(commit, latest: nil)
       idx_hash = self.get_idx
       idx_hash[:commit] = commit
@@ -789,5 +805,18 @@ module Cnvrg
       end
       Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/set_started", "POST", {job_type: job_type, job_id: job_id})
     end
+
+    def self.stop_if_project_present(project_home, project_name)
+      cli = Cnvrg::CLI.new()
+      config = YAML.load_file(project_home + "/.cnvrg/config.yml")
+      local_commit = YAML.load_file(project_home + "/.cnvrg/idx.yml")[:commit] rescue nil
+      return if local_commit.blank?
+      if config[:project_name] == project_name
+        cli.log_message("Project already present, clone aborted")
+        exit(0)
+      end
+    rescue => e
+      nil
+    end
   end
 end

data/lib/cnvrg/ssh.rb

@@ -1,6 +1,5 @@
 require 'fileutils'
 require 'cnvrg/files'
-require 'docker'
 require 'net/ssh'
 
 

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.6.38'
+  VERSION = '1.9.5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.6.38
+  version: 1.9.5
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-23 00:00:00.000000000 Z
+date: 2020-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -210,14 +210,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.11.417
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.11.417
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: signet
   requirement: !ruby/object:Gem::Requirement
@@ -344,20 +344,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.22.1
-- !ruby/object:Gem::Dependency
-  name: docker-api
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.33'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.33'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -386,20 +372,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: net-ssh
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: down
   requirement: !ruby/object:Gem::Requirement
@@ -427,6 +399,7 @@ files:
 - lib/cnvrg.rb
 - lib/cnvrg/Images.rb
 - lib/cnvrg/api.rb
+- lib/cnvrg/api_v2.rb
 - lib/cnvrg/auth.rb
 - lib/cnvrg/cli.rb
 - lib/cnvrg/cli/flow.rb
@@ -434,6 +407,7 @@ files:
 - lib/cnvrg/cli/subcommand.rb
 - lib/cnvrg/cli/task.rb
 - lib/cnvrg/colors.rb
+- lib/cnvrg/connect_job_ssh.rb
 - lib/cnvrg/data.rb
 - lib/cnvrg/datafiles.rb
 - lib/cnvrg/dataset.rb
@@ -445,11 +419,13 @@ files:
 - lib/cnvrg/files.rb
 - lib/cnvrg/flow.rb
 - lib/cnvrg/helpers.rb
+- lib/cnvrg/helpers/agent.rb
 - lib/cnvrg/helpers/executer.rb
 - lib/cnvrg/hyper.rb
 - lib/cnvrg/image.rb
 - lib/cnvrg/image_cli.rb
 - lib/cnvrg/job_cli.rb
+- lib/cnvrg/job_ssh.rb
 - lib/cnvrg/logger.rb
 - lib/cnvrg/org_helpers.rb
 - lib/cnvrg/project.rb
@@ -477,7 +453,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.