cnvrg 1.9.9.9.7

data/lib/cnvrg/downloader/client.rb

@@ -0,0 +1,101 @@
+
+module Cnvrg
+  module Downloader
+    OLD_SERVER_VERSION_MESSAGE = "Your server version is not relevant for this cli version please contact support for further help."
+    MAXIMUM_BACKOFF = 64
+    RETRIES = ENV['UPLOAD_FILE_RETRIES'].try(:to_i) || 20
+    attr_accessor :bucket, :client
+    class Client
+      def initialize(params)
+        @key = ''
+        @iv = ''
+        @client = ''
+        @bucket = ''
+      end
+
+      def extract_key_iv(sts_path)
+        sts = open(sts_path).read rescue nil
+        raise StandardError.new("Cant open sts") if sts.blank?
+        sts.split("\n")
+      end
+
+      def cut_prefix(prefix, file)
+        file.gsub(prefix, '').gsub(/^\/*/, '')
+      end
+
+      def download(storage_path, local_path)
+        ### need to be implemented..
+      end
+
+      def upload(storage_path, local_path)
+        ### need to be implemented..
+      end
+
+      def mkdir(path, recursive: false)
+        recursive ? FileUtils.mkdir_p(path) : FileUtils.mkdir(path)
+      end
+
+      def prepare_download(local_path)
+        mkdir(File.dirname(local_path), recursive: true)
+      end
+
+      def decrypt(str)
+        Cnvrg::Helpers.decrypt(@key, @iv, str)
+      end
+
+      def safe_upload(storage_path, local_path)
+        n = 1
+        error = nil
+        while n <= RETRIES
+          begin
+            self.upload(storage_path, local_path)
+            error = nil
+            break
+          rescue => e
+            backoff_time_seconds = backoff_time(n)
+
+            message = "Got error: #{e.class.name} with message: #{e.message} while uploading a single file: #{local_path}, retry: #{n} of: #{RETRIES}"
+            if n < RETRIES
+              message += ", next retry in: #{backoff_time_seconds} seconds"
+            else
+              message += ", done retry, continuing to the next file"
+            end
+            Cnvrg::Logger.log_error_message(message)
+
+            sleep backoff_time_seconds
+
+            n += 1
+            error = e
+          end
+        end
+        raise error if error.present?
+        true
+      end
+
+      def self.factory(params)
+        params = params.as_json
+        case params["storage"]
+        when 's3', 'minio'
+          return Cnvrg::Downloader::Clients::S3Client.new(sts_path: params["path_sts"], access_key: params["sts_a"], secret: params["sts_s"], session_token: params["sts_st"], region: params["region"], bucket: params["bucket"], encryption: params["encryption"], endpoint: params["endpoint"], storage: params["storage"])
+        when 'azure'
+          azure_params = params.symbolize_keys.slice(*[:storage_account_name, :storage_access_key, :container, :sts])
+          return Cnvrg::Downloader::Clients::AzureClient.new(**azure_params)
+        when 'gcp'
+          return Cnvrg::Downloader::Clients::GcpClient.new(project_id: params["project_id"], credentials: params["credentials"], bucket_name: params["bucket_name"], sts: params["sts"])
+        end
+      end
+
+      private
+
+      def random_number_milliseconds
+        rand(1000) / 1000.0
+      end
+
+
+      def backoff_time(n)
+        return [((2**n)+random_number_milliseconds), MAXIMUM_BACKOFF].min
+      end
+
+    end
+  end
+end

data/lib/cnvrg/downloader/clients/azure_client.rb

@@ -0,0 +1,45 @@
+require 'azure/storage/blob'
+
+module Cnvrg
+  module Downloader
+    module Clients
+      class AzureClient < Client
+        def initialize(storage_account_name: nil, storage_access_key: nil, container: nil, sts: nil)
+          @key, @iv = extract_key_iv(sts)
+          @account_name = Cnvrg::Helpers.decrypt(@key, @iv, storage_account_name)
+          @access_key = Cnvrg::Helpers.decrypt(@key, @iv, storage_access_key)
+          @container = Cnvrg::Helpers.decrypt(@key, @iv, container)
+        end
+
+        def download(storage_path, local_path, decrypt: true)
+          prepare_download(local_path)
+          storage_path = Cnvrg::Helpers.decrypt(@key, @iv, storage_path) if decrypt
+          blob, content = client.get_blob(@container, storage_path)
+          ::File.open(local_path, 'wb') {|f| f.write(content)}
+          blob
+        end
+
+        def upload(storage_path, local_path)
+          begin
+            client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))
+          rescue => e
+            raise e
+          end
+        end
+
+        def fetch_files(prefix: nil, marker: nil, limit: 10000)
+          blobs = client.list_blobs(@container, prefix: prefix, max_results: limit, marker: marker)
+          next_marker = blobs.continuation_token
+          files = blobs.map{|x| x.name}
+          [files, next_marker]
+        end
+
+
+        private
+        def client
+          Azure::Storage::Blob::BlobService.create(storage_account_name: @account_name, storage_access_key: @access_key)
+        end
+      end
+    end
+  end
+end

data/lib/cnvrg/downloader/clients/gcp_client.rb

@@ -0,0 +1,50 @@
+require "google/cloud/storage"
+
+module Cnvrg
+  module Downloader
+    module Clients
+      class GcpClient < Client
+        def initialize(project_id: nil, credentials: nil, bucket_name: nil, sts: nil)
+          @key, @iv = extract_key_iv(sts)
+          @project_id = Cnvrg::Helpers.decrypt(@key, @iv, project_id)
+          @credentials_path = Cnvrg::Helpers.decrypt(@key, @iv, credentials)
+          @tempfile = nil
+          @bucket_name = Cnvrg::Helpers.decrypt(@key, @iv, bucket_name)
+          init_gcp_credentials
+          @storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials, retries: 20)
+          @bucket = @storage.bucket(@bucket_name)
+          @bucket.name
+        rescue => e
+          Cnvrg::Logger.log_error(e)
+          Cnvrg::Logger.log_info("Tried to init gcp client without success.")
+          Cnvrg::CLI.log_message("Cannot init client. please contact support to check your bucket credentials.")
+          exit(1)
+        end
+
+        def init_gcp_credentials
+          t = Tempfile.new
+          f = open(@credentials_path).read
+          t.binmode
+          t.write(f)
+          t.rewind
+          @credentials = t.path
+          @tempfile = t
+        end
+
+        def download(storage_path, local_path)
+          prepare_download(local_path)
+          file = @bucket.file(decrypt(storage_path))
+          file.download local_path
+        end
+
+        def upload(storage_path, local_path)
+          begin
+            @bucket.create_file(local_path, storage_path)
+          rescue => e
+            raise e
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cnvrg/downloader/clients/s3_client.rb

@@ -0,0 +1,78 @@
+module Cnvrg
+  module Downloader
+    module Clients
+      class S3Client < Client
+        def initialize(sts_path: nil, access_key: nil, secret: nil, session_token: nil, region: nil, bucket: nil, encryption: nil, endpoint: nil, storage: nil)
+          @key, @iv = extract_key_iv(sts_path)
+          @access_key = Cnvrg::Helpers.decrypt(@key, @iv, access_key)
+          @secret = Cnvrg::Helpers.decrypt(@key, @iv, secret)
+          @session_token = Cnvrg::Helpers.decrypt(@key, @iv, session_token)
+          @region = Cnvrg::Helpers.decrypt(@key, @iv, region)
+          @bucket_name = Cnvrg::Helpers.decrypt(@key, @iv, bucket)
+          @endpoint = Cnvrg::Helpers.decrypt(@key, @iv, endpoint)
+          options = {
+              :access_key_id => @access_key,
+              :secret_access_key => @secret,
+              :session_token => @session_token,
+              :region => @region,
+              :http_open_timeout => 60, :retry_limit => 20
+          }
+          if storage == 'minio'
+            options.delete(:session_token)
+            options = options.merge({
+                                        :force_path_style => true,
+                                        :ssl_verify_peer => false,
+                                        :endpoint => @endpoint,
+                                    })
+          end
+
+          @options = options
+
+          #@client = Aws::S3::Client.new(options)
+          #@bucket = Aws::S3::Resource.new(client: @client).bucket(@bucket_name)
+          @upload_options = {:use_accelerate_endpoint => storage == 's3'}
+          if encryption.present?
+            @upload_options[:server_side_encryption] = encryption
+          end
+        end
+
+        def download(storage_path, local_path, decrypt: true)
+          prepare_download(local_path)
+          storage_path = Cnvrg::Helpers.decrypt(@key, @iv, storage_path) if decrypt
+          resp = nil
+          File.open(local_path, 'w+') do |file|
+            resp = aws_client.get_object({bucket: @bucket_name, key: storage_path}, target: file)
+          end
+          resp
+        rescue => e
+          Cnvrg::Logger.log_error(e)
+          raise e
+        end
+
+        def upload(storage_path, local_path)
+          ### storage path is the path inside s3 (after the bucket)
+          # local path is fullpath for the file /home/ubuntu/user.../hazilim.py
+          o = aws_bucket.object(storage_path)
+          success = o.upload_file(local_path, @upload_options)
+          return success
+        rescue => e
+          raise e
+        end
+
+        def fetch_files(prefix: nil, marker: nil, limit: 1000)
+          batch_files = aws_bucket.objects(prefix: prefix, marker: marker).first(limit)
+          batch_files.to_a.map(&:key)
+        end
+
+        private
+        def aws_client
+          Aws::S3::Client.new(@options)
+        end
+
+        def aws_bucket
+          Aws::S3::Resource.new(client: aws_client).bucket(@bucket_name)
+        end
+      end
+    end
+  end
+end

data/lib/cnvrg/experiment.rb

@@ -0,0 +1,209 @@
+require 'net/http'
+module Cnvrg
+  class Experiment
+    attr_reader :slug
+    attr_reader :sync_before_terminate
+    attr_reader :sync_delay_time
+    attr_reader :output_dir
+
+    def initialize(owner, project_slug, job_id: nil)
+      @project_slug = project_slug
+      @owner = owner
+      @command = nil
+      @base_resource = "users/#{owner}/projects/#{project_slug}/"
+      @slug = job_id
+      @sync_before_terminate = nil
+      @sync_delay_time = nil
+      @output_dir = nil
+    end
+
+    def as_env
+      return {
+          CNVRG_JOB_ID: @slug,
+          CNVRG_JOB_TYPE: "Experiment",
+          CNVRG_PROJECT: @project_slug,
+          CNVRG_OWNER: @owner,
+      }.as_json
+    end
+
+    def start(input, platform, machine_name, start_commit, name, email_notification, machine_activity,script_path,
+              sync_before_terminate, periodic_sync)
+
+      res = Cnvrg::API.request(@base_resource + "experiment/start", 'POST',
+                               {input: input, platform: platform, machine_name: machine_name, start_commit: start_commit,
+                                title: name, email_notification: email_notification, machine_activity: machine_activity,script_path:script_path})
+      Cnvrg::CLI.is_response_success(res,false)
+
+      @slug = res.to_h["result"].to_h["slug"]
+      @sync_before_terminate = res.to_h["result"].to_h["sync_before_terminate"]
+      @sync_delay_time = res.to_h["result"].to_h["sync_delay_time"]
+      @output_dir = res.to_h["result"].to_h["output_dir"]
+      @command = res.to_h["result"].to_h["command"] rescue nil
+
+      return res
+
+    end
+
+    def start_notebook_session(kernel, start_commit, token, port, remote, notebook_path)
+      res = Cnvrg::API.request(@base_resource + "notebook/start_session", 'POST',
+                               {kernel: kernel, start_commit: start_commit,
+                                 token: token, port: port, remote: remote, notebook_path: notebook_path})
+      Cnvrg::CLI.is_response_success(res)
+
+      @slug = res["result"]["id"]
+
+
+      return @slug
+
+    end
+
+    def end_notebook_session(notebook_slug)
+      res = Cnvrg::API.request(@base_resource + "notebook/#{notebook_slug}/stop", 'GET')
+      Cnvrg::CLI.is_response_success(res,false)
+
+      return res
+
+    end
+    def update_notebook_slug(proj_dir, slug)
+      begin
+      file = proj_dir+"/.cnvrg/notebook_slug"
+      FileUtils.touch file
+
+      File.open(file, "w+") { |f| f.write slug }
+      rescue
+      end
+
+    end
+
+    def get_notebook_slug(proj_dir)
+      begin
+        notebook_slug = File.open(proj_dir + "/.cnvrg/notebook_slug", "rb").read
+        notebook_slug = notebook_slug.gsub("/n", "")
+        notebook_slug = notebook_slug.to_s.strip
+        return notebook_slug
+      rescue
+        return nil
+      end
+
+
+    end
+    def get_machine_activity(working_dir)
+      begin
+        machine_activity = File.open("#{working_dir}/.cnvrg/machine_activity", "rb").read
+        machine_activity = machine_activity.to_s.strip
+        ma_id = machine_activity.to_i
+        return ma_id
+      rescue
+        return nil
+      end
+
+
+    end
+
+    def job_log(logs, level: 'info', step: nil, job_type: nil, job_id: nil)
+      logs = [logs].flatten
+      logs.each_slice(10).each do |temp_logs|
+        Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/jobs/experiment/#{@slug}/log", "POST", {job_type: "Experiment", job_id: @slug, logs: temp_logs, log_level: level, step: step, timestamp: Time.now})
+        sleep(1)
+      end
+    end
+
+    def exec_remote(command, commit_to_run, instance_type, image_slug,schedule,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync,
+                    sync_before_terminate, max_time, ds_sync_options=0,output_dir=nil,data_query=nil,
+                    git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil, prerun: true, requirements: true, recurring: nil,
+                    email_notification_error: false, email_notification_success: false, emails_to_notify: nil)
+      response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
+                                                                                                            commit_sha1: commit_to_run,
+                                                                                                            instance_type: instance_type,
+                                                                                                            schedule:schedule,
+                                                                                                            local_timestamp:local_timestamp,
+                                                                                                            datasets: datasets,
+                                                                                                            grid: grid,
+                                                                                                            path_to_cmd:path_to_cmd,dataset_slug:data,
+                                                                                                            dataset_commit: data_commit,max_time:max_time,
+                                                                                                            periodic_sync:periodic_sync, sync_before_terminate:sync_before_terminate,
+                                                                                                            dataset_sync_options:ds_sync_options,output_dir:output_dir,
+                                                                                                            dataset_query:data_query,git_commit:git_commit,git_branch:git_branch,
+                                                                                                            restart_if_stuck:restart_if_stuck, local_folders: local_folders, title:title,
+                                                                                                            prerun: prerun, requirements: requirements, recurring: recurring,
+                                                                                                            email_notification_error: email_notification_error, email_notification_success: email_notification_success,
+                                                                                                            emails_to_notify: emails_to_notify})
+
+      return response
+    end
+    def remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options=0,data_query=nil, image = nil, datasets = nil)
+      response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/notebook/remote", 'POST', {instance_type: instance_type,dataset_slug:data,
+                                                                                                          dataset_commit: data_commit,image_slug:image,
+                                                                                                          datasets: datasets,
+                                                                                                          commit:commit,notebook_type:notebook_type,dataset_sync_options:ds_sync_options,
+                                                                                                          dataset_query:data_query})
+      return response
+    end
+
+    def upload_temp_log(temp_log)
+      response = Cnvrg::API.request(@base_resource + "experiment/upload_temp_log", 'POST', {output: temp_log,
+                                                                                            exp_slug: @slug})
+      Cnvrg::CLI.is_response_success(response,false)
+    end
+
+    def send_machine_stats(stats)
+      response = Cnvrg::API.request(@base_resource + "experiment/upload_stats", "POST", {exp_slug: @slug, stats: stats.map{|s| s.merge!({time: Time.now})}})
+      Cnvrg::CLI.is_response_success(response,false)
+    end
+
+    def end(output, exit_status, end_commit, cpu_average, memory_average, end_time: nil)
+      #if remote try to remove
+      tries = 0
+      success = false
+      end_time ||= Time.now
+      while tries < 10 and success.blank?
+        sleep (tries*rand) ** 2 ### exponential backoff
+                                ## this call is super important so we cant let it crash.
+
+        tries += 1
+        response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', {output: output, exp_slug: @slug,
+                                                                                exit_status: exit_status, end_commit: end_commit,
+                                                                                cpu_average: cpu_average, memory_average: memory_average, end_time: end_time})
+        success = Cnvrg::CLI.is_response_success(response,false)
+      end
+
+      begin
+        FileUtils.rm_rf(["/home/ds/.cnvrg/tmp/exec.log"])
+      rescue
+
+      end
+    end
+
+    def get_cmd
+      return @command
+    end
+
+    def restart_spot_instance
+
+      restart = false
+      begin
+      url = URI.parse('http://169.254.169.254/latest/meta-data/spot/termination-time')
+      req = Net::HTTP::Get.new(url.to_s)
+      res = Net::HTTP.start(url.host, url.port) {|http|
+        http.request(req)
+      }
+      unless res.body.include? "404"
+        restart = true
+      end
+      if res.body.include? "Empty reply from server"
+        restart = false
+      end
+      rescue
+        restart = false
+
+      end
+
+      return restart
+
+    end
+
+    def send_restart_request(sha1=nil)
+      Cnvrg::API.request(@base_resource + "experiment/check_spot_instance", 'POST', {exp_slug: @slug, end_commit: sha1})
+    end
+  end
+end