cnvrg 1.6.38 → 1.9.5

data/lib/cnvrg/dataset.rb

@@ -7,7 +7,7 @@ module Cnvrg
     IDXParallelThreads ||= Cnvrg::Helpers.parallel_threads
     IDXParallelProcesses ||= Parallel.processor_count
 
-    def initialize(project_home = '', dataset_url: '')
+    def initialize(project_home = '', dataset_url: '', dataset_info: '')
       begin
         @info = {}
         if project_home.present?
@@ -17,6 +17,11 @@ module Cnvrg
           @title = config[:dataset_name]
           @slug = config[:dataset_slug]
           @owner = config[:owner]
+        elsif dataset_info.present?
+          @title = dataset_info[:slug]
+          @slug = dataset_info[:slug]
+          @owner = dataset_info[:owner]
+          @local_path = Dir.pwd
         else
           owner, slug = Cnvrg::Helpers.extract_owner_slug_from_url(dataset_url, 'datasets')
           @title = slug
@@ -87,7 +92,6 @@ module Cnvrg
       files
     end
 
-
     def get_stats(commit: nil, query: nil)
       response = Cnvrg::API.request("users/#{@owner}/datasets/#{@slug}/clone", 'POST', {commit: commit, query: query})
       Cnvrg::CLI.is_response_success(response, true)
@@ -100,7 +104,6 @@ module Cnvrg
       response['result']['files']['keys']
     end
 
-
     def backup_idx
       Cnvrg::Logger.log_info("Backup idx")
       if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
@@ -123,7 +126,6 @@ module Cnvrg
       File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
     end
 
-
     def self.delete(dataset_slug, owner)
       response = Cnvrg::API.request("users/#{owner}/datasets/#{dataset_slug}/delete", 'DELETE')
       return response
@@ -142,14 +144,12 @@ module Cnvrg
       response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/volumes/create", 'POST', {data_commit: commit})
       CLI.is_response_success(response)
       return response
-
     end
 
     def list(owner)
       response = Cnvrg::API.request("users/#{owner}/datasets/list", 'GET')
       CLI.is_response_success(response)
       return response
-
     end
 
     def search_queries
@@ -160,7 +160,6 @@ module Cnvrg
         row << [query["name"], query["slug"], query["created_at"].in_time_zone.to_s, query["username"]]
       end
       return row
-
     end
 
     def get_query_file(query_slug)
@@ -171,7 +170,6 @@ module Cnvrg
         row << [file["name"], file["fullpath"], file["s3_url"]]
       end
       return row
-
     end
 
     def download_tags_yaml
@@ -186,11 +184,13 @@ module Cnvrg
       end
     end
 
-    def list_commits
-      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/list_commits", 'GET')
+    def list_commits(commit_sha1: nil)
+      response = Cnvrg::API.request(
+          "users/#{self.owner}/datasets/#{self.slug}/list_commits?commit=#{commit_sha1}",
+          'GET'
+      )
       CLI.is_response_success(response)
       return response
-
     end
 
     def upload_tags_via_yml(tag_file = nil)
@@ -277,7 +277,6 @@ module Cnvrg
         end
       end
       return ignore_list.flatten
-
     end
 
 
@@ -293,7 +292,6 @@ module Cnvrg
         create_ignore = true
       end
 
-
       cnvrgignore = Helpers.cnvrgignore_content
       begin
         response = Cnvrg::API.request("cli/create_dataset", 'POST', {title: dataset_name, owner: owner, is_public: is_public, bucket: bucket})
@@ -351,14 +349,14 @@ module Cnvrg
     end
 
     def self.blank_clone(owner, dataset_name, dataset_slug)
-      list_dirs = [".cnvrg"
+      list_dirs = ["#{dataset_slug}/.cnvrg"
       ]
       list_files = [
-          ".cnvrg/config.yml"
+          "#{dataset_slug}/.cnvrg/config.yml"
       ]
       create_ignore = false
       if !File.exist? ".cnvrgignore"
-        list_files << ".cnvrgignore"
+        list_files << "#{dataset_slug}/.cnvrgignore"
         create_ignore = true
       end
 
@@ -372,8 +370,8 @@ module Cnvrg
 
         FileUtils.mkdir_p list_dirs
         FileUtils.touch list_files
-        File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
-        File.open(".cnvrgignore", "w+") {|f| f.write cnvrgignore} unless !create_ignore
+        File.open("#{dataset_slug}/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
+        File.open("#{dataset_slug}/.cnvrgignore", "w+") {|f| f.write cnvrgignore} unless !create_ignore
       rescue => e
         return false
       end
@@ -385,16 +383,21 @@ module Cnvrg
       File.open(self.local_path + "/.cnvrgignore", "w+") {|f| f.write cnvrgignore}
     end
 
-    def self.verify_datasets(dataset_titles, timeout = 100)
+    def self.verify_dataset(dataset_slug)
+      config = YAML.load_file("/data/#{dataset_title}/.cnvrg/config.yml") rescue {}
+      config[:success] == true
+    end
+
+    def self.verify_datasets(dataset_titles, timeout = nil)
       start_time = Time.now.to_i
       Cnvrg::Logger.log_info("Verifying datasets #{dataset_titles}")
       Cnvrg::Logger.log_info("Timeout is  #{timeout}")
       while true
         begin
           current_time = Time.now.to_i
-          return false if timeout < current_time - start_time
+          return false if (timeout.present? and timeout < current_time - start_time)
           all_are_ready = dataset_titles.all? do |dataset_title|
-            config = YAML.load_file("/data/#{dataset_title}/.cnvrg/config.yml")
+            config = YAML.load_file("#{dataset_title}/.cnvrg/config.yml")
             config[:success] == true
           end
           return true if all_are_ready
@@ -408,6 +411,30 @@ module Cnvrg
       end
     end
 
+    def self.scan_datasets()
+      Cnvrg::Logger.log_info("Looking up datasets")
+      datasets = Dir.entries(Dir.pwd).map do |entry|
+        if File.directory? File.join(Dir.pwd,entry) and !(entry =='.' || entry == '..')
+          begin
+            config = YAML.load_file("#{Dir.pwd}/#{entry}/.cnvrg/config.yml") rescue nil
+            local_commit = YAML.load_file("#{Dir.pwd}/#{entry}/.cnvrg/idx.yml")[:commit] rescue nil
+            if config.present? and config[:success] == true and config[:dataset_name].present? and config[:dataset_slug].present? and local_commit.present?
+              {
+                  "dataset_slug": config[:dataset_slug],
+                  "dataset_name": config[:dataset_name],
+                  "local_commit": local_commit,
+              }
+            else
+              nil
+            end
+          rescue
+            nil
+          end
+        end
+      end.compact.uniq
+      datasets
+    end
+
     def clone(commit)
       return
     end
@@ -470,7 +497,6 @@ module Cnvrg
       if File.exist?(file_path)
         File.open(file_path, "a") {|f| f.puts(":success: true")}
       end
-    rescue
     end
 
     def self.init_container(owner, dataset_slug, dataset_name)
@@ -504,7 +530,6 @@ module Cnvrg
         return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
       end
       {commit: nil, tree: {}}
-
     end
 
     def set_idx(idx)
@@ -572,7 +597,7 @@ module Cnvrg
     end
 
     def write_idx(tree = nil, commit = nil)
-      if tree.blank?
+      if tree.nil?
         tree = self.generate_idx[:tree]
         tree = tree.map {|k, v| (v.present?) ? [k, {sha1: v[:sha1], commit_time: Time.now}] : [k, v]}.to_h
       end
@@ -701,7 +726,6 @@ module Cnvrg
       idx.try(:fetch, :partial_commit)
     end
 
-
     def current_status(new_branch)
       commit = last_local_commit
       response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status_current", 'POST', {current_commit: commit, new_branch: new_branch})
@@ -731,7 +755,6 @@ module Cnvrg
       rescue
         return []
       end
-
     end
 
 
@@ -748,7 +771,6 @@ module Cnvrg
         idx_hash[:tree] = ""
       else
         idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
-
       end
       idx_hash[:next_commit] = commit_sha1
       File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
@@ -810,11 +832,9 @@ module Cnvrg
 
     def update_idx(idx)
       File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx.to_yaml}
-
       return true
     end
 
-
     def update_idx_with_commit!(commit)
       idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
       idx_hash[:commit] = commit
@@ -852,5 +872,21 @@ module Cnvrg
       return {validation: Data::ConfigValidation::SUCCESS, message: "Directory is already linked to #{slug}"}
     end
 
+    def self.stop_if_dataset_present(dataset_home, dataset_name, commit: nil)
+
+      cli = Cnvrg::CLI.new()
+      config = YAML.load_file(dataset_home + "/.cnvrg/config.yml")
+      if commit.present?
+        local_commit = YAML.load_file(dataset_home + "/.cnvrg/idx.yml")[:commit] rescue nil
+        return if commit != local_commit or local_commit.blank?
+      end
+      if config[:dataset_name] == dataset_name
+        cli.log_message("Dataset already present, clone aborted")
+        exit(0)
+      end
+    rescue => e
+      nil
+    end
+
   end
 end

data/lib/cnvrg/experiment.rb

@@ -6,11 +6,12 @@ module Cnvrg
     attr_reader :sync_delay_time
     attr_reader :output_dir
 
-    def initialize(owner, project_slug)
+    def initialize(owner, project_slug, job_id: nil)
       @project_slug = project_slug
       @owner = owner
+      @command = nil
       @base_resource = "users/#{owner}/projects/#{project_slug}/"
-      @slug = nil
+      @slug = job_id
       @sync_before_terminate = nil
       @sync_delay_time = nil
       @output_dir = nil
@@ -37,6 +38,8 @@ module Cnvrg
       @sync_before_terminate = res.to_h["result"].to_h["sync_before_terminate"]
       @sync_delay_time = res.to_h["result"].to_h["sync_delay_time"]
       @output_dir = res.to_h["result"].to_h["output_dir"]
+      @command = res.to_h["result"].to_h["command"] rescue nil
+
       return res
 
     end
@@ -123,8 +126,7 @@ module Cnvrg
                                                                                                             dataset_query:data_query,git_commit:git_commit,git_branch:git_branch,
                                                                                                             restart_if_stuck:restart_if_stuck, local_folders: local_folders, title:title,
                                                                                                             prerun: prerun, requirements: requirements, recurring: recurring,
-                                                                                                            email_notification_error: email_notification_error,
-                                                                                                            email_notification_success: email_notification_success,
+                                                                                                            email_notification_error: email_notification_error, email_notification_success: email_notification_success,
                                                                                                             emails_to_notify: emails_to_notify})
 
       return response
@@ -172,6 +174,10 @@ module Cnvrg
       end
     end
 
+    def get_cmd
+      return @command
+    end
+
     def restart_spot_instance
 
       restart = false

data/lib/cnvrg/files.rb

@@ -86,7 +86,7 @@ module Cnvrg
       end
     end
 
-    def upload_multiple_files(files_list, commit_sha1, progress: nil)
+    def upload_multiple_files(files_list, commit_sha1, progress: nil, suppress_exceptions: false)
       #open files on the server.
       Cnvrg::Logger.log_info("Uploading project files")
       return if files_list.blank?
@@ -95,7 +95,7 @@ module Cnvrg
         return self.upload_files_old(files_list, commit_sha1, progress: progress)
       end
 
-      files_list = files_list.map{|x| [x,self.parse_file(x)]}.to_h
+      files_list = files_list.map{|x| [x, self.parse_file(x)] if self.parse_file(x)}.compact.to_h
       resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST', {files: files_list, commit: commit_sha1})
       unless Cnvrg::CLI.is_response_success(resp, false)
         raise SignalException.new("Cant upload files to the server.")
@@ -106,6 +106,7 @@ module Cnvrg
 
       #upload files
       token_mutex = Mutex.new
+      buffered_errors = {}
       blob_ids = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
 
         token_mutex.synchronize {
@@ -115,21 +116,33 @@ module Cnvrg
         begin
           Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
         rescue => e
-
-          Cnvrg::CLI.log_message("Failed to upload #{file}: #{e.message}", 'red')
+          Cnvrg::CLI.log_message("Failed to upload #{file}: #{e.message}", 'red') unless suppress_exceptions
           Cnvrg::Logger.log_error(e)
           Cnvrg::Logger.log_method(bind: binding)
-          raise e
+
+          buffered_errors[file] = "Failed to upload #{file}: #{e.message}" if suppress_exceptions
+
+          raise e unless suppress_exceptions
         end
         progress.progress += 1 if progress.present?
-        files[file]["bv_id"]
+
+        unless buffered_errors.key?(file)
+          files[file]["bv_id"]
+        else
+          nil
+        end
       end
 
+      # remove nil files (failed files) from blob_ids
+      blob_ids.compact!
+
       #save files on the server.
       resp = Cnvrg::API.request(@base_resource + "upload_files_save", 'POST', {blob_ids: blob_ids, commit: commit_sha1})
       unless Cnvrg::CLI.is_response_success(resp, false)
         raise SignalException.new("Cant save uploaded files to the server.")
       end
+
+      return buffered_errors
     end
 
 
@@ -146,7 +159,7 @@ module Cnvrg
       end
     end
 
-    def delete_files_from_server(files, commit_sha1)
+    def delete_files_from_server(files, commit_sha1, suppress_exceptions: false)
       #files are absolute path files here. ^^
       if Cnvrg::Helpers.server_version < 1
         return self.delete_files_from_server_old(files, commit_sha1)
@@ -156,8 +169,14 @@ module Cnvrg
       return if files.blank?
       resp = Cnvrg::API.request(@base_resource + "delete_files", 'DELETE', {files: files, commit: commit_sha1})
       unless Cnvrg::CLI.is_response_success(resp, false)
-        raise SignalException.new("Cant delete the following files from the server.")
+        raise SignalException.new("Cant delete the following files from the server.") unless suppress_exceptions
+        Cnvrg::Logger.log_error_message("Cant delete the following files from the server: ")
+        Cnvrg::Logger.log_error_message(files.to_s)
       end
+    rescue => e
+      Cnvrg::Logger.log_error_message("An exception raised in delete_files_from_server: ")
+      Cnvrg::Logger.log_error(e)
+      raise e unless suppress_exceptions
     end
 
     def upload_single_file(file)
@@ -176,6 +195,8 @@ module Cnvrg
       sha1 =  OpenSSL::Digest::SHA1.file(abs_path).hexdigest
 
       {relative_path: file, absolute_path: abs_path, file_name: file_name, file_size: file_size, content_type: content_type, sha1: sha1}
+    rescue => e
+        return false
     end
 
     def upload_old(absolute_path, relative_path, commit_sha1)
@@ -706,6 +727,7 @@ module Cnvrg
       unless Cnvrg::CLI.is_response_success(res, false)
         raise SignalException.new("Cant download files from the server.")
       end
+      puts res
       self.download_multpile_files_s3(res['result'], @project_home, postfix: postfix, progress: progress)
     end
 
@@ -911,12 +933,22 @@ module Cnvrg
       end
     end
 
-    def start_commit(new_branch,force:false, exp_start_commit:nil, job_slug: nil, job_type: nil, start_commit: nil, message: nil)
-      response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug, new_branch: new_branch,force:force,
-                                                                              username: @owner, exp_start_commit:exp_start_commit, job_slug: job_slug, job_type: job_type, start_commit: start_commit, message: message})
-      Cnvrg::CLI.is_response_success(response,false)
-      return response
-    end
+    def start_commit(new_branch,force:false, exp_start_commit:nil, job_slug: nil, job_type: nil, start_commit: nil, message: nil, debug_mode: false)
+        response = Cnvrg::API.request(
+            "#{base_resource}/commit/start",
+            'POST',
+            {
+                project_slug: @project_slug, username: @owner,
+                new_branch: new_branch, force:force,
+                exp_start_commit:exp_start_commit, start_commit: start_commit,
+                job_slug: job_slug, job_type: job_type, message: message,
+                debug_mode: debug_mode
+            }
+        )
+
+        Cnvrg::CLI.is_response_success(response,false)
+        return response
+      end
 
     def end_commit(commit_sha1,force:false,message:"")
       response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force,message:message})

data/lib/cnvrg/helpers.rb

@@ -359,9 +359,9 @@ parameters:
       return {client: client, key: key, iv: iv, bucket: bucket, upload_options: upload_options}
     end
 
-    def get_experiment_events_log_from_server(exp, project)
+    def get_experiment_events_log_from_server(exp, project, commit: nil)
       dest_dir = exp["slug"]
-      commit = exp["end_commit"]
+      commit = commit || exp["end_commit"]
       response = project.clone(0, commit)
       Cnvrg::CLI.is_response_success(response, should_exit=false)
       commit_sha1 = response["result"]["commit"]
@@ -372,42 +372,50 @@ parameters:
       FileUtils.rm_rf("#{dest_dir}")
       FileUtils.mkdir_p(dest_dir)
       files.each do |f|
-        FileUtils.mv(f, "#{dest_dir}/#{File.basename(f)}")
+        file_dir = "#{dest_dir}/#{File.dirname(f)}"
+        FileUtils.mkdir_p(file_dir)
+        FileUtils.mv(f, "#{dest_dir}/#{f}")
       end
     end
 
     def get_experiment_events_log_via_kubectl(exp, namespace)
       dest_dir = exp["slug"]
-      result = `kubectl -n #{namespace} get pods | grep #{exp["machine_activity"]}`
+      result = `kubectl -n #{namespace} get pods | grep #{exp["slug"]}`
+
       pod_name = result.split(" ")[0]
-      if pod_name.present?
-        FileUtils.mkdir_p(dest_dir)
-        working_dir = `kubectl -n #{namespace} exec #{pod_name} -- pwd`
-        working_dir.strip!
-        res = `kubectl -n #{namespace} exec #{pod_name} -- /bin/bash -c "ls -R #{working_dir}"`
-        files_and_folders = res.split("\n\n")
-        all_files = []
-
-        files_and_folders.each do |file_and_folder|
-          files = file_and_folder.split("\n")
-          if files.first.include?(":")
-            folder = files.first.gsub(":", "")
-            files = files.drop(1)
-          end
-          files.each do |file|
-            if file.include?("tfevents")
-              all_files << "#{folder}/#{file}"
-            end
-          end
+      return false if pod_name.blank?
+      FileUtils.mkdir_p(dest_dir)
+      working_dir = `kubectl -n #{namespace} exec #{pod_name} -c agent -- pwd`
+      working_dir.strip!
+      res = `kubectl -n #{namespace} exec #{pod_name} -c agent -- /bin/bash -c "ls -R #{working_dir}"`
+      files_and_folders = res.split("\n\n")
+      all_files = []
+
+      files_and_folders.each do |file_and_folder|
+        files = file_and_folder.split("\n")
+        if files.first.include?(":")
+          folder = files.first.gsub(":", "")
+
+          folder = folder.sub(working_dir + "/", "")
+          files = files.drop(1)
         end
-
-        all_files.each do |file|
-          res = `kubectl -n #{namespace} cp #{pod_name}:#{file} #{dest_dir}/#{File.basename(file)}`
+        files.each do |file|
+          if file.include?("tfevents")
+            all_files << "#{folder}/#{file}"
+          end
         end
+      end
 
+      all_files.each do |file|
+        file_dir = "#{dest_dir}/#{File.dirname(file)}"
+        FileUtils.mkdir_p(file_dir)
+        res = `kubectl -n #{namespace} cp #{pod_name}:#{file} -c agent #{dest_dir}/#{file}`
       end
+
+      return true
     rescue => e
       Cnvrg::Logger.log_error(e)
+      return false
     end
   end