cnvrg 1.2.7 → 1.3.2

data/lib/cnvrg/downloader/client.rb

@@ -0,0 +1,53 @@
+
+module Cnvrg
+  module Downloader
+    OLD_SERVER_VERSION_MESSAGE = "Your server version is not relevant for this cli version please contact support for further help."
+    attr_accessor :bucket, :client
+    class Client
+      def initialize(params)
+        @key = ''
+        @iv = ''
+        @client = ''
+        @bucket = ''
+      end
+
+      def extract_key_iv(sts_path)
+        sts = open(sts_path).read rescue nil
+        raise StandardError.new("Cant open sts") if sts.blank?
+        sts.split("\n")
+      end
+
+      def download(storage_path, local_path)
+        ### need to be implemented..
+      end
+
+      def upload(storage_path, local_path)
+        ### need to be implemented..
+      end
+
+      def mkdir(path, recursive: false)
+        recursive ? FileUtils.mkdir_p(path) : FileUtils.mkdir(path)
+      end
+
+      def prepare_download(local_path)
+        mkdir(File.dirname(local_path), recursive: true)
+      end
+
+      def decrypt(str)
+        Cnvrg::Helpers.decrypt(@key, @iv, str)
+      end
+
+      def self.factory(params)
+        params = params.as_json
+        case params["storage"]
+        when 's3', 'minio'
+          return Cnvrg::Downloader::Clients::S3Client.new(sts_path: params["path_sts"], access_key: params["sts_a"], secret: params["sts_s"], session_token: params["sts_st"], region: params["region"], bucket: params["bucket"], encryption: params["encryption"], endpoint: params["endpoint"], storage: params["storage"])
+        when 'azure'
+          return Cnvrg::Downloader::Clients::AzureDownloader.new(params)
+        when 'gcp'
+          return Cnvrg::Downloader::Clients::GcpClient.new(project_id: params["project_id"], credentials: params["credentials"], bucket_name: params["bucket_name"], sts: params["sts"])
+        end
+      end
+    end
+  end
+end

data/lib/cnvrg/downloader/clients/azure_client.rb

@@ -0,0 +1,22 @@
+module Cnvrg
+  module Downloader
+    module Clients
+      class AzureClient < Client
+        def initialize(storage: nil, account_name: nil, access_key: nil, container: nil, sts_path: nil)
+          @key, @iv = extract_key_iv(sts_path)
+          @account_name = Cnvrg::Helpers.decrypt(@key, @iv, account_name)
+          @access_key = Cnvrg::Helpers.decrypt(@key, @iv, access_key)
+          @container = Cnvrg::Helpers.decrypt(@key, @iv, container)
+        end
+
+        def download(storage_path, local_path)
+
+        end
+
+        def upload(storage_path, local_path)
+
+        end
+      end
+    end
+  end
+end

data/lib/cnvrg/downloader/clients/gcp_client.rb

@@ -0,0 +1,46 @@
+require "google/cloud/storage"
+
+module Cnvrg
+  module Downloader
+    module Clients
+      class GcpClient < Client
+        def initialize(project_id: nil, credentials: nil, bucket_name: nil, sts: nil)
+          @key, @iv = extract_key_iv(sts)
+          @project_id = Cnvrg::Helpers.decrypt(@key, @iv, project_id)
+          @credentials_path = Cnvrg::Helpers.decrypt(@key, @iv, credentials)
+          @tempfile = nil
+          @bucket_name = Cnvrg::Helpers.decrypt(@key, @iv, bucket_name)
+          init_gcp_credentials
+          @storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials)
+          @bucket = @storage.bucket(@bucket_name)
+          @bucket.name
+        rescue => e
+          Cnvrg::Logger.log_error(e)
+          Cnvrg::Logger.log_info("Tried to init gcp client without success.")
+          Cnvrg::CLI.log_message("Cannot init client. please contact support to check your bucket credentials.")
+          exit(1)
+        end
+
+        def init_gcp_credentials
+          t = Tempfile.new
+          f = open(@credentials_path).read
+          t.binmode
+          t.write(f)
+          t.rewind
+          @credentials = t.path
+          @tempfile = t
+        end
+
+        def download(storage_path, local_path)
+          prepare_download(local_path)
+          file = @bucket.file(decrypt(storage_path))
+          file.download local_path
+        end
+
+        def upload(storage_path, local_path)
+          @bucket.create_file(local_path, storage_path)
+        end
+      end
+    end
+  end
+end

data/lib/cnvrg/downloader/clients/s3_client.rb

@@ -0,0 +1,51 @@
+module Cnvrg
+  module Downloader
+    module Clients
+      class S3Client < Client
+        def initialize(sts_path: nil, access_key: nil, secret: nil, session_token: nil, region: nil, bucket: nil, encryption: nil, endpoint: nil, storage: nil)
+          @key, @iv = extract_key_iv(sts_path)
+          @access_key = Cnvrg::Helpers.decrypt(@key, @iv, access_key)
+          @secret = Cnvrg::Helpers.decrypt(@key, @iv, secret)
+          @session_token = Cnvrg::Helpers.decrypt(@key, @iv, session_token)
+          @region = Cnvrg::Helpers.decrypt(@key, @iv, region)
+          @bucket_name = Cnvrg::Helpers.decrypt(@key, @iv, bucket)
+          @endpoint = Cnvrg::Helpers.decrypt(@key, @iv, endpoint)
+          options = {
+              :access_key_id => @access_key,
+              :secret_access_key => @secret,
+              :session_token => @session_token,
+              :region => @region,
+              :http_open_timeout => 60, :retry_limit => 20
+          }
+          if storage == 'minio'
+            options[:endpoint] = @endpoint
+          end
+
+          @client = Aws::S3::Client.new(options)
+          @bucket = Aws::S3::Resource.new(client: @client).bucket(@bucket_name)
+          @upload_options = {:use_accelerate_endpoint => storage == 's3'}
+          if encryption.present?
+            @upload_options[:server_side_encryption] = encryption
+          end
+        end
+
+        def download(storage_path, local_path)
+          prepare_download(local_path)
+          storage_path = Cnvrg::Helpers.decrypt(@key, @iv, storage_path)
+          File.open(local_path, 'w+') do |file|
+            resp = @client.get_object({bucket: @bucket_name,
+                                key: storage_path}, target: file)
+          end
+          resp
+        end
+
+        def upload(storage_path, local_path)
+          ### storage path is the path inside s3 (after the bucket)
+          # local path is fullpath for the file /home/ubuntu/user.../hazilim.py
+          o = @bucket.object(storage_path)
+          o.upload_file(local_path, @upload_options)
+        end
+      end
+    end
+  end
+end

data/lib/cnvrg/files.rb

@@ -18,6 +18,10 @@ module Cnvrg
       @base_resource = "users/#{owner}/projects/#{project_slug}/"
       @project_home = project_home.presence || Cnvrg::CLI.get_project_home
       @project = project
+      @client = nil
+      if @project.present?
+        @client = @project.get_storage_client
+      end
       @progressbar = progressbar
       @custom_progess = false
       @cli = cli
@@ -84,16 +88,10 @@ module Cnvrg
       # resolve bucket
       res = resp['result']
       files = res['files']
-      props = Cnvrg::Helpers.get_s3_props(res)
-      client = props[:client]
-      bucket = props[:bucket]
-      upload_options = props[:upload_options]
-      s3_bucket = Aws::S3::Resource.new(client: client).bucket(bucket)
-
       #upload files
       # files.keys.map do |file|
       Parallel.map(files.keys, self.get_upload_options) do |file|
-        resp = Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]), s3_bucket, options: upload_options)}
+        resp = Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
         raise SignalException.new("Cant upload #{file}") unless resp
         progress.progress += 1 if progress.present?
       end
@@ -134,16 +132,10 @@ module Cnvrg
       end
     end
 
-    def upload_single_file(file, bucket, options: {})
+    def upload_single_file(file)
       path = file['path']
       absolute_path = file[:absolute_path]
-
-      resp = bucket.object(path).
-          upload_file(absolute_path, options)
-      unless resp
-        raise SignalException.new("Cant upload #{absolute_path}")
-      end
-      resp
+      @client.upload(path, absolute_path)
     end
 
     def parse_file(file)
@@ -192,15 +184,9 @@ module Cnvrg
       upload_resp = Cnvrg::API.request("/users/#{@owner}/" + "upload_cli_log", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
                                                                                              file_name: file_name, log_date: log_date,
                                                                                              file_size: file_size, file_content_type: content_type})
-      if Cnvrg::CLI.is_response_success(upload_resp, false)
-       s3_res = upload_large_files_s3(upload_resp, absolute_path)
-
-      end
-      if s3_res
-        return true
-      end
-      return false
-
+      path, client = upload_resp["path"], upload_resp["client"]
+      @client = Cnvrg::Downloader::Client.factory(client)
+      @client.upload(path, absolute_path)
     end
 
     def upload_exec_file(absolute_path, image_name, commit_id)
@@ -711,7 +697,7 @@ module Cnvrg
 
     def download_multpile_files_s3(files, project_home, postfix: '', progress: nil)
       begin
-        props = Cnvrg::Helpers.get_s3_props(files)
+        props = {}
         client = props[:client]
         iv = props[:iv]
         key = props[:key]
@@ -739,20 +725,11 @@ module Cnvrg
               if not File.exists?(project_home+"/"+File.dirname(file_path))
                 FileUtils.makedirs(project_home+"/"+File.dirname(file_path))
               end
-
-              file_key =  Cnvrg::Helpers.decrypt(key,iv, f["path"])
-              resp = false
-              Cnvrg::Helpers.try_until_success(tries: 10) {
-                File.open(project_home+"/"+file_path, 'w+') do |file|
-                  resp = client.get_object({bucket:bucket,
-                                          key:file_key}, target: file)
-                end
-              }
+              local_path = project_home+"/"+file_path
+              storage_path = f["path"]
+              @client.download(storage_path, local_path)
               progress.progress += 1 if progress.present?
               download_succ_count += 1
-
-
-
             rescue => e
               return Cnvrg::Result.new(false,"Could not create file: #{file_path}", e.message, e.backtrace)
               raise Parallel::Kill
@@ -913,12 +890,8 @@ module Cnvrg
     end
 
     def download_file(file_path: '', key: '', iv: '', bucket: '', path: '', client: nil)
-      file_key =  Cnvrg::Helpers.decrypt(key,iv, path)
-      resp = false
-      File.open(@project_home+"/"+file_path, 'w+') do |file|
-        resp = client.get_object({bucket:bucket,
-                                  key:file_key}, target: file)
-      end
+      local_path = @project_home+"/"+file_path
+      @client.download(path, local_path)
     end
 
     def delete(file)
@@ -936,7 +909,7 @@ module Cnvrg
     def handle_compare_idx(compared, resolver: {})
       begin
         all_files = compared.values.flatten.uniq
-        props = Cnvrg::Helpers.get_s3_props(resolver)
+        props = {}
         files = resolver['keys'].map{|f| [f['name'], f]}.to_h
         client = props[:client]
         iv = props[:iv]

data/lib/cnvrg/helpers/executer.rb

@@ -0,0 +1,171 @@
+class Cnvrg::Helpers::Executer
+  def initialize(project: nil, job_type: nil, job_id: nil)
+    @project = project
+    @job_type = job_type
+    @job_id = job_id
+    if job_id.blank?
+      Cnvrg::CLI.log_message("Cant find job, exiting.", 'red')
+      exit(1)
+    end
+  end
+
+  def fetch_commands(block: nil, key: nil)
+    resp = Cnvrg::API.request("#{base_url}/commands", "GET", {block: block})
+    commands = decrypt_commands(resp["commands"], resp["key"], resp["iv"])
+    commands.map{|k| k.with_indifferent_access}
+  end
+
+  def decrypt_commands(text,key,iv)
+    text, key, iv = [text,key,iv].map{|x| x.unpack('m')[0]}
+    decipher = OpenSSL::Cipher::AES256.new :CBC
+    decipher.decrypt
+    decipher.key = key
+    decipher.iv = iv
+    commands = decipher.update(text) + decipher.final
+    JSON.parse(commands)
+  end
+
+  def execute(cmd)
+    ## execute the command for running
+    # cmd will have to following fields
+    #
+    # :command => the command to execute
+    # :type => the command type, 'notify' or nil
+    # :timeout => the timeout for the command in seconds (default is 60 hours)
+    # :retries => integer, default 1
+    #
+    # when type == 'file_exists'
+    #  'file' => string => file to check (fullpath)
+    #  'exists_commands' => list of commands in case file exists
+    #   'non_exists_commands' => list of commands in case file doesnt exists
+    # when type == 'notify'
+    # :before_execute_log => log to be logged before execution
+    # :logs => boolean => add the execution logs to the job logs
+    # :title => command title, can replace the on_error, on_success fields
+    # :on_error_log => log to be logged on exit_code != 0
+    # :on_success_log => log to be logged on exit_code == 0
+    #
+    retries = cmd[:retries] || 1
+    resp = []
+    retries.times.each do
+      resp = execute_helper(cmd)
+      exit_status, _, _, _, _ = resp
+      return resp if exit_status == 0
+    end
+    return resp
+  end
+
+  def execute_cmds(cmds)
+    cmds.each do |command|
+      puts "===================="
+      puts "Execute #{command[:command]}"
+      execute(command)
+      puts "===================="
+    end
+  end
+
+
+  private
+  def execute_helper(cmd)
+    case cmd[:type]
+    when 'notify'
+      return run_and_notify(cmd)
+    when 'file_exists'
+      if File.exists? cmd[:file]
+        return execute_cmds(cmd[:exists_commands]) if cmd[:exists_commands].present?
+      else
+        return execute_cmds(cmd[:non_exists_commands]) if cmd[:non_exists_commands].present?
+      end
+    when 'create_file'
+      return create_file(cmd)
+    else
+      return regular_command(cmd)
+    end
+  end
+
+  def run_and_notify(cmd)
+    with_logs = cmd[:logs]
+    cmd = init_cmd_logs(cmd)
+    job_log(cmd[:before_execute_log]) if cmd[:before_execute_log].present?
+    exit_status, output, errors, start_time, end_time = regular_command(cmd)
+    logs = []
+    if exit_status == 0
+      logs = output.map{|log| log[:logs]} if with_logs
+      job_log(logs + cmd[:on_success_log])
+    else
+      logs = output + errors
+      logs = logs.sort_by{|x| x[:timestamp]}.map{|x| x[:logs]} if with_logs
+      job_log(logs + cmd[:on_error_log], level: 'error')
+    end
+    return [exit_status, output, errors, start_time, end_time]
+  end
+
+  def merge_log_block(logs)
+    logs.group_by {|log| log[:timestamp].to_s}
+        .map {|ts, logz| {timestamp: ts, logs: logz.map {|l| l[:log]}.join("\n")}}
+  end
+
+
+  def regular_command(cmd = {})
+    errors = []
+    output = []
+    start_time = Time.now
+    timeout = cmd[:timeout] || 60*60*60
+    t = Thread.new do
+      PTY.spawn(cmd[:command]) do |stdout, stdin, pid, stderr|
+        begin
+          stdout.each do |line|
+            puts line
+            output << {log: line.strip, timestamp: Time.now}
+          end
+          if stderr.present?
+            stderr.each do |line|
+              errors << {log: line.strip, timestamp: Time.now}
+            end
+          end
+        rescue => e
+          errors << {log: e.message, timestamp: Time.now}
+        end
+        ::Process.wait pid
+      end
+    end
+    while t.status
+      if Time.now - start_time > timeout
+        puts "Kill thread because of timeout..."
+        errors << {log: "Timeout", timestamp: Time.now}
+        Thread.kill(t)
+      end
+      sleep 1
+    end
+    exit_status = $?.exitstatus
+    end_time = Time.now
+    [exit_status, merge_log_block(output), merge_log_block(errors), start_time, end_time]
+  end
+
+  def create_file(cmd)
+    cmd = init_cmd_logs(cmd)
+    File.open(cmd[:path], "w+"){|f| f.write(cmd[:content])}
+  end
+
+  def init_cmd_logs(cmd)
+    if cmd[:title]
+      cmd[:before_execute_log] ||= ["Running #{cmd[:title]}"]
+      cmd[:on_error_log] ||= ["#{cmd[:title]} was failed during running"]
+      cmd[:on_success_log] ||= ["#{cmd[:title]} executed successfully"]
+    end
+    cmd[:on_success_log] ||= []
+    cmd[:on_error_log] ||= []
+    [:before_execute_log, :on_success_log, :on_error_log].each{|x| cmd[x] = [cmd[x]].flatten}
+    cmd
+  end
+
+  def base_url
+    "users/#{@project.owner}/projects/#{@project.slug}/jobs/#{@job_type.underscore}/#{@job_id}"
+  end
+
+
+  def job_log(logs, level: 'info', step: nil)
+    @project.job_log(logs, level: level, step: step, job_type: @job_type, job_id: @job_id)
+  end
+
+end