cnvrg 1.6.35 → 1.9.8

data/lib/cnvrg/connect_job_ssh.rb

@@ -0,0 +1,31 @@
+module Cnvrg
+  class ConnectJobSsh
+    def initialize(job_id)
+      home_dir = File.expand_path('~')
+      config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+      @owner = config.to_h[:owner]
+      @job_id = job_id
+    rescue => e
+      @owner = ""
+      Cnvrg::Logger.log_info("cnvrg is not configured")
+    end
+
+    def start(username, password)
+      Cnvrg::API_V2.request("#{@owner}/job_ssh/#{@job_id}/start" , 'POST', {username: username, password: password})
+    end
+
+    def status()
+      Cnvrg::API_V2.request("#{@owner}/job_ssh/#{@job_id}/status" , 'GET', nil)
+    end
+
+    def run_portforward_command(pod_name, port, kubeconfig, namespace)
+      command = "kubectl"
+      if kubeconfig.present?
+        command = "kubectl --kubeconfig=#{kubeconfig}"
+      end
+      bashCommand = "#{command} -n #{namespace} port-forward #{pod_name} #{port}:22"
+      puts("\nrunning command #{bashCommand}")
+      `#{bashCommand}`
+    end
+  end
+end

data/lib/cnvrg/data.rb

@@ -128,6 +128,7 @@ module Cnvrg
       message = options["message"]
       cli.sync_data_new(new_branch, force, verbose, commit, all_files, tags, parallel, chunk_size, init, message)
     end
+
     desc 'data download', 'Download files from remote server'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits", :default => false
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
@@ -151,27 +152,52 @@ module Cnvrg
     method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
     method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
     method_option :relative, :type => :boolean, :aliases => ["-rel", "--relative"], :default => false
-
+    method_option :flatten, :type => :boolean, :aliases => ["-f", "--flatten"], :default => false
+    method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
     def clone(dataset_url)
-      #test
       cli = Cnvrg::CLI.new()
       only_tree =options[:only_tree]
       commit =options[:commit]
       query =options[:query]
       read = options[:read]
       remote = options[:remote]
-      cli.clone_data(dataset_url, only_tree=only_tree,commit=commit, query=query, read=read, remote=remote, relative: options[:relative])
+      soft = options[:soft]
+      flatten = options[:flatten]
+      cli.clone_data(
+          dataset_url,
+          only_tree=only_tree,
+          commit=commit,
+          query=query,
+          read=read,
+          remote=remote,
+          flatten: flatten,
+          relative: options[:relative],
+          soft: soft
+      )
     end
 
     desc 'data verify DATASETS_TITLES', 'verify datasets', :hide => true
-    method_option :timeout, :type => :numeric, :aliases => ["-t", "--timeout"], :desc => "Time to wait before returning final answer", :default => 15
-
+    method_option :timeout, :type => :numeric, :aliases => ["-t", "--timeout"], :desc => "Time to wait before returning final answer", :default => nil
     def verify(*dataset_titles)
       cli = Cnvrg::CLI.new()
       timeout =options[:timeout]
       cli.verify_datasets(dataset_titles, timeout)
     end
 
+    desc 'data scan', 'lookup datasets', :hide => true
+    def scan()
+      cli = Cnvrg::CLI.new()
+      cli.scan_datasets()
+    end
+
+    desc "data block DATASET_TITLES", 'verifying that datasets exists', hide: true
+    def block(*dataset_slugs)
+      not_verified = true
+      while not_verified
+        not_verified = dataset_slugs.select{|slug| not Dataset.verify_dataset(slug)}.present?
+      end
+    end
+
     desc 'data set --url=DATASET_URL', 'Set dataset url to other url'
     method_option :url, :type => :string, :aliases => ["--url"], :default => ''
     def set
@@ -186,22 +212,48 @@ module Cnvrg
     desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
     method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
     method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
+    method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
+    method_option :threads, :type => :numeric, :aliases => ["-t","--threads"], :default => 15
+    method_option :chunk_size, :type => :numeric, :aliases => ["-cs","--chunk"], :default => 1000
     method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
     def put(dataset_url, *files)
       cli = Cnvrg::CLI.new()
       dir = options[:dir]
+      force = options["force"]
       commit = options[:commit]
       message = options[:message]
+      threads = options[:threads]
+      chunk_size = options[:chunk_size]
+      cli.data_put(
+          dataset_url,
+          files: files,
+          dir: dir,
+          commit: commit,
+          force: force,
+          threads: threads,
+          chunk_size: chunk_size,
+          message: message
+      )
+    end
 
-      cli.data_put(dataset_url, files: files, dir: dir, commit: commit, message: message)
+    desc 'data rm DATASET_URL FILES_PREFIX', 'Delete selected files from remote server'
+    method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
+    def rm(dataset_url, *regex_list)
+      cli = Cnvrg::CLI.new()
+      message = options[:message]
+      cli.data_rm(dataset_url, regex_list: regex_list, message: message)
     end
 
     desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'
     method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
+    method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
+    method_option :flatten, :type => :boolean, :aliases => ["-f", "--flatten"], :default => false
     def clone_query(dataset_url)
       cli = Cnvrg::CLI.new()
-      query =options[:query]
-      cli.clone_data_query(dataset_url,query=query)
+      query = options[:query]
+      flatten = options[:flatten]
+      soft =options[:soft]
+      cli.clone_data_query(dataset_url,query=query, flatten, soft: soft)
     end
 
     desc 'data delete DATASET_SLUG', 'Delete dataset'
@@ -217,12 +269,13 @@ module Cnvrg
       cli.list_dataset()
 
     end
-    desc 'data commits', 'List all commits for a current dataset'
 
-    def commits()
+    desc 'data commits URL/SLUG', 'List all commits for a given dataset'
+    method_option :commit_sha1, :type => :string, :aliases => ["-c", "--commit"], :default => nil
+    def commits(dataset_url)
       cli = Cnvrg::CLI.new()
-      cli.list_dataset_commits()
-
+      commit_sha1 = options[:commit_sha1]
+      cli.list_dataset_commits(dataset_url, commit_sha1:commit_sha1)
     end
 
     desc 'data files DATASET_URL', 'Show list of dataset files'

data/lib/cnvrg/datafiles.rb

@@ -1,5 +1,5 @@
 require 'mimemagic'
-require 'aws-sdk'
+require 'aws-sdk-s3'
 require 'URLcrypt'
 require 'parallel'
 require 'fileutils'
@@ -25,7 +25,7 @@ module Cnvrg
 
     def refresh_storage_token
       current_time = Time.current
-      if current_time - @token_issue_time > 3.hours
+      if current_time - @token_issue_time > 1.hours
         @downloader = @dataset.get_storage_client
         @token_issue_time = Time.current
       end
@@ -55,6 +55,26 @@ module Cnvrg
       paths.map{|p| p.gsub(/^\.\//, '')}
     end
 
+    def get_files_and_folders(paths)
+      files_and_folders = {}
+      paths.each do |file|
+        if File.exists? file
+          if File.directory? file
+            Dir.glob("#{file}/**/*").select do |f|
+              files_and_folders["#{f}/"] = "folder" if File.directory? f
+              files_and_folders[f] = "file" if File.file? f
+            end
+            files_and_folders["#{file}/"] = "folder"
+          else
+            files_and_folders[file] = "file"
+          end
+          next
+        end
+        raise SignalException.new(1, "Cant find file #{file}") unless File.exists? "#{Dir.pwd}/#{file}"
+      end
+      return files_and_folders
+    end
+
     def check_files_sha1(files, resolver, tag)
       conflicts = 0
       files.each do |file|
@@ -110,18 +130,36 @@ module Cnvrg
       end
     end
 
-
+    # This is for backwards compatibility only and should be removed in future versions:
     def put_commit(commit_sha1)
-      response = Cnvrg::API.request("#{@base_resource}/commit/latest", 'PUT', {commit_sha1: commit_sha1})
+      response = Cnvrg::API.request(
+          "#{@base_resource}/commit/latest",
+          'PUT',
+          {
+              commit_sha1: commit_sha1,
+              ignore: true # tells the new server to ignore this api call since its coming from the new CLI
+          }
+      )
       if response.present?
         msg = response['result']
       else
-        msg =  "Cant save changes in the dataset"
+        msg =  "Can't save changes in the dataset"
       end
 
       Cnvrg::Result.new(Cnvrg::CLI.is_response_success(response, false), msg)
     end
 
+    def create_progressbar(title, total)
+      return ProgressBar.create(
+          :title => title,
+          :progress_mark => '=',
+          :format => "%b>>%i| %p%% %t",
+          :starting_at => 0,
+          :total => total,
+          :autofinish => true
+      )
+    end
+
 
     def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, prefix: '', partial_commit: nil, total: nil)
       begin
@@ -158,6 +196,9 @@ module Cnvrg
         end
 
         files = results['files']
+
+        progressbar.progress += tree.keys.length - files.length if progressbar.present?
+        progress_semaphore = Mutex.new
         upload_error_files = []
         @temp_upload_progressbar.progress += tree.keys.length - files.length if @temp_upload_progressbar.present?
         Parallel.map((files.keys), in_threads: threads) do |k|
@@ -169,8 +210,9 @@ module Cnvrg
             tree.except!(k)
             Cnvrg::Logger.log_error_message("Error while upload single file #{o["path"]}")
           end
-          @temp_upload_progressbar.progress += 1 if @temp_upload_progressbar.present?
+          progress_semaphore.synchronize { progressbar.progress += 1 if progressbar.present? }
         end
+
         blob_ids = files.values.map {|f| f['bv_id']}
         if blob_ids.present?
           dirs = tree.keys.select {|k| tree[k].nil?} || []
@@ -190,17 +232,231 @@ module Cnvrg
       end
     end
 
-    def upload_single_file(file)
+    def delete_multiple_files(commit_sha1, regex_list)
+      begin
+        Cnvrg::Logger.log_info("Sending Delete Files request")
+        resp = Cnvrg::API.request(
+            @base_resource + "delete_files",
+            'POST_JSON',
+            {
+                commit_sha1: commit_sha1,
+                regex_list: regex_list,
+            }
+        )
+        unless Cnvrg::CLI.is_response_success(resp, false)
+          Cnvrg::Logger.log_method(bind: binding)
+          raise Exception.new("Got an error message from server, #{resp.try(:fetch, "message")}")
+        end
+        Cnvrg::Logger.log_info("Delete Files request Successful")
+        return resp["files"], resp["folders"], resp["job_id"]
+      rescue => e
+        Cnvrg::Logger.log_method(bind: binding)
+        Cnvrg::Logger.log_error(e)
+        raise e
+      end
+    end
+
+    def delete_file_chunk(commit_sha1, regex_list, chunk_size, offset)
+      begin
+        resp = Cnvrg::API.request(
+            @base_resource + "delete_files_by_chunk",
+            'POST_JSON',
+            {
+                commit_sha1: commit_sha1,
+                regex_list: regex_list,
+                chunk_size: chunk_size,
+                offset: offset
+            }
+        )
+        unless Cnvrg::CLI.is_response_success(resp, false)
+          Cnvrg::Logger.log_method(bind: binding)
+          raise Exception.new("Got an error message from server, #{resp.try(:fetch, "message")}")
+        end
+        return resp["total_changes"]
+      rescue => e
+        Cnvrg::Logger.log_method(bind: binding)
+        Cnvrg::Logger.log_error(e)
+        raise e
+      end
+    end
+
+    def get_delete_progress(commit_sha1, job_id)
+      begin
+        resp = Cnvrg::API.request(
+            @base_resource + "get_delete_progress",
+            'POST_JSON',
+            {
+                commit_sha1: commit_sha1,
+                job_id: job_id
+            }
+        )
+        unless Cnvrg::CLI.is_response_success(resp, false)
+          Cnvrg::Logger.log_method(bind: binding)
+          raise Exception.new("Got an error message from server, #{resp.try(:fetch, "message")}")
+        end
+        return resp["total_deleted"]
+      rescue => e
+        Cnvrg::Logger.log_method(bind: binding)
+        Cnvrg::Logger.log_error(e)
+        raise e
+      end
+    end
+
+    def request_upload_files(commit_sha1, tree, force, new_branch, partial_commit)
+      retry_count = 0
+      loop do
+        upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {
+            commit_sha1: commit_sha1,
+            tree: tree,
+            force: force,
+            is_branch: new_branch,
+            partial_commit: partial_commit
+        })
+        if not (Cnvrg::CLI.is_response_success(upload_resp, false))
+          #Cnvrg::Logger.log_method(bind: binding)
+          retry_count += 1
+
+          puts "Failed request upload files: #{Time.current}"
+          puts upload_resp
+
+          if retry_count > 5
+            raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+          end
+          sleep 5
+          next
+        end
+        return upload_resp['result'].with_indifferent_access
+      end
+
+    end
+
+    def upload_multiple_files_optimized(files, commit_sha1, threads: ParallelThreads, chunk_size: 1000, force: false, new_branch: false, prefix: '', partial_commit: nil)
+      cli = CLI.new
+      cli.log_message("Using #{threads} threads with chunk size of #{chunk_size}.", Thor::Shell::Color::GREEN)
+
+      progressbar = create_progressbar("Upload Progress", files.size)
+      cli = CLI.new
+
+      # Vars to handle the parallelism
+      progress_mutex = Mutex.new
+      file_queue = Queue.new
+      progress_queue = Queue.new
+      worker_threads = []
+
+      # Vars to keep track of uploaded files and directories
+      dirs = []
+      uploaded_files = []
+
+      begin
+        # Init working threads that handle the upload of the files:
+        threads.times do |i|
+          worker_threads[i] = Thread.new do
+            # wait for file_queue.close to break the loop
+            while file = file_queue.deq
+              success = upload_single_file(cli, file)
+              file[:success] = success
+              if not success
+                cli.log_message("Error while uploading file: #{file[:absolute_path]}", Thor::Shell::Color::RED)
+                Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
+              end
+              progress_queue << file
+            end
+          end
+        end
+
+        # init the thread that handles the file upload progress and saving them in the server
+        progress_thread = Thread.new do
+          loop do
+            file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
+            unless file.nil?
+              progress_mutex.synchronize {
+                progressbar.progress += 1
+                uploaded_files.append(file) if file[:success]
+              }
+
+              if uploaded_files.size == chunk_size or progressbar.finished?
+                refresh_storage_token
+                # puts "progress: #{progress_queue.length}"
+                # puts "files: #{file_queue.length}"
+                Cnvrg::Logger.info("Sending Upload files save")
+                blob_ids = uploaded_files.map {|f| f['bv_id']}
+                upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
+                unless Cnvrg::CLI.is_response_success(upload_resp, false)
+                  Cnvrg::Logger.log_method(bind: binding)
+                  raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+                end
+                # cli.log_message("Saved file chunk to server", Thor::Shell::Color::GREEN)
+                uploaded_files = []
+                dirs = []
+              end
+            else
+              sleep(0.1)
+            end
+
+            if progressbar.finished?
+              # puts "finished"
+              file_queue.close()
+              progress_queue.close()
+              Thread.exit
+            end
+          end
+        end
+
+        file_chunks = files.each_slice(chunk_size).to_a
+        # Fetch the required files from the server:
+        Parallel.map((file_chunks), in_threads: 10) do |files_chunk|
+
+          tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix)
+          results = request_upload_files(commit_sha1, tree, force, new_branch, partial_commit)
+
+          # puts "Got #{results['files'].size} files to upload from #{files_chunk.size} files"
+
+          if results['files'].blank?
+            progress_mutex.synchronize { progressbar.progress += tree.keys.length }
+            next
+          end
+
+          # Handle directories:
+          new_dirs = tree.keys.select {|k| tree[k].nil?}
+          dirs += new_dirs
+
+          files_to_upload = results['files']
+          progress_mutex.synchronize {
+            progressbar.progress += tree.keys.length - files_to_upload.length
+          }
+
+          files_to_upload.keys.each do |key|
+            while file_queue.size > 5000
+              sleep(0.1)
+            end
+            file_queue.push tree[key].merge(files_to_upload[key])
+          end
+        end
+
+        progress_thread.join()
+        worker_threads.each(&:join)
+
+      rescue => e
+        puts e
+        Cnvrg::Logger.log_method(bind: binding)
+        Cnvrg::Logger.log_error(e)
+        raise e
+      end
+    end
+
+    def upload_single_file(cli, file)
+      success = false
       begin
         file = file.as_json
         Cnvrg::Logger.log_info("Uploading #{file["absolute_path"]}")
         @downloader.safe_upload(file["path"], file["absolute_path"])
+        success = true
         Cnvrg::Logger.log_info("#{file["absolute_path"]} uploaded.")
       rescue => e
         Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
         Cnvrg::Logger.log_error(e)
-        return false
       end
+      success
     end
 
     def upload_file(absolute_path, relative_path, commit_sha1)
@@ -869,45 +1125,57 @@ module Cnvrg
         return false
       end
     end
+
     def revoke_download(tar_files, extracted_files)
       begin
-
         FileUtils.rm_rf(tar_files) unless (tar_files.nil? or tar_files.empty?)
         FileUtils.rm_rf(extracted_files) unless (extracted_files.nil? or extracted_files.empty?)
-
       rescue => e
         return false
       end
-
       return true
-
     end
+
     def delete_commit(commit_sha1)
       response = Cnvrg::API.request("#{base_resource}/commit/#{commit_sha1}", 'DELETE')
       Cnvrg::CLI.is_response_success(response, true)
       return response
     end
+
     def get_commit(commit_sha1)
       response = Cnvrg::API.request("#{base_resource}/commit/#{commit_sha1}", 'GET')
       Cnvrg::CLI.is_response_success(response, true)
       return response
     end
 
-    def start_commit(new_branch,force=false,delete_commit=nil, chunks: 0, dataset: @dataset, message:nil)
+    def start_commit(new_branch, force=false, chunks: 0, dataset: @dataset, message:nil)
       begin
         #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
         # create a new commit.
         idx = (force || new_branch) ? {} : dataset.get_idx
         commit = idx[:commit]
         next_commit = idx[:next_commit]
-        response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
-                                                                                username: @owner,current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
-        Cnvrg::CLI.is_response_success(response, true)
-        return response
+        response = Cnvrg::API.request(
+            "#{base_resource}/commit/start",
+            'POST',
+            {
+                dataset_slug: @dataset_slug,
+                new_branch: new_branch,
+                force:force,
+                username: @owner,
+                current_commit: commit,
+                next_commit: next_commit,
+                total_chunks: chunks,
+                message: message
+            }
+        )
+      Cnvrg::CLI.is_response_success(response, true)
+      return response
       rescue => e
         return false
       end
     end
+
     def last_valid_commit()
       begin
         #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
@@ -920,15 +1188,24 @@ module Cnvrg
       end
     end
 
-    def end_commit(commit_sha1,force, success: true, uploaded_files: 0 )
+    def end_commit(commit_sha1, force, success: true, uploaded_files: 0, commit_type: nil)
       begin
-        response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force, success: success, uploaded_files: uploaded_files})
+        response = Cnvrg::API.request(
+            "#{base_resource}/commit/end",
+            'POST',
+            {
+                commit_sha1: commit_sha1,
+                force:force,
+                success: success,
+                uploaded_files: uploaded_files,
+                commit_type: commit_type
+            }
+        )
         Cnvrg::CLI.is_response_success(response, true)
         return response
       rescue => e
         return false
       end
-
     end
 
     def end_commit_tar(commit_sha1, cur_idx)
@@ -1011,11 +1288,17 @@ module Cnvrg
       end
     end
 
-    def generate_parallel_idx
-      
+    def last_valid_commit()
+      begin
+        response = Cnvrg::API.request("#{base_resource}/last_valid_commit", 'GET')
+        Cnvrg::CLI.is_response_success(response, true)
+        return response
+      rescue => e
+        return false
+      end
     end
 
-    def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false)
+    def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false, flatten: false)
       begin
         refresh_storage_token
         parallel_options = {
@@ -1024,22 +1307,24 @@ module Cnvrg
         }
         Parallel.map(files["keys"], parallel_options) do |f|
           begin
-          local_path = @dataset.local_path + '/' + f['name']
-          Cnvrg::Logger.log_info("Downloading #{local_path}")
-          progressbar.progress += 1 if progressbar.present?
-          if local_path.end_with? "/"
-            @downloader.mkdir(local_path, recursive: true)
-            next
-          end
-            # blob
-          local_path = "#{local_path}.conflict" if conflict
-          storage_path = f["path"]
-          if File.exists? local_path
-            Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
-            next
-          end
-          resp = @downloader.download(storage_path, local_path)
-          Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
+            file_path = f['name']
+            file_path = File.basename(f['name']) if flatten
+            local_path = @dataset.local_path + '/' + file_path
+            Cnvrg::Logger.log_info("Downloading #{local_path}")
+            progressbar.progress += 1 if progressbar.present?
+            if local_path.end_with? "/"
+              @downloader.mkdir(local_path, recursive: true)
+              next
+            end
+              # blob
+            local_path = "#{local_path}.conflict" if conflict
+            storage_path = f["path"]
+            if File.exists? local_path
+              Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
+              next
+            end
+            resp = @downloader.download(storage_path, local_path)
+            Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
           rescue => e
             Cnvrg::Logger.log_error(e)
           end