cnvrg 2.0.20 → 2.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d5e7aa6c49c6bbe9ce99dcc85f90825d67ef8a4d066c84dd5978da99afb116b
-  data.tar.gz: 81e5d10c09beddd7da049ac29d2bd71fc611f3c2ca5bd4b2e2431457573b683d
+  metadata.gz: 296eba8c8dab87e1a16b7980c1e80b013be95af52cfc184a12cf366f676a3a2b
+  data.tar.gz: '08ad62abd898bb6bb1a9099237f5ebd854f87359ecc571036ce351b259127d78'
 SHA512:
-  metadata.gz: c338755e158c6e1c03dc84c16f3015c1801875d98e58ad788da47b5cf2374b06fa62445279ddd9585b9e2dd3a62bee381af968f1e5925c2b74a9eedf0652b17b
-  data.tar.gz: 674fe36d75e00c70919067c2539a85c3c56c6850852faf00781f8419a3a3a1dabb3d925be56316a9d20903a869300b761a893f01a0cea94eef6703f99c34d5a8
+  metadata.gz: f5174c705ed765c76538401ea14ea2c14e267d98836b3d8e8b66a72d31d4f7ee2eeb313fbd080cd15a513bb4149f907aef075be2d7f6bab4b96afefb4ba5f341
+  data.tar.gz: 355b440e9a009571e2f097599d14418fcdbe0a79cfaae8473637722745923f8b138b4fdbc0e3ee60107122835f98468811661b770ffee4008f8b141250361fce

data/Readme.md

@@ -80,4 +80,22 @@
 * DEV-13271 - Bug: CLI - on upload folders in working dir containing .cnvrg, dir not uploading - dir is on .cnvrgignore
 ## Version v2.0.20
 2022-02-27
-* DEV-12288 - Bug: wrong error message when upload fails
+* DEV-12288 - Bug: wrong error message when upload fails
+## Version v2.1.1
+2022-05-01
+## Version v2.1.2
+2022-05-08
+* DEV-13815 - Bug: CLI - remove "cnvrg data sync" command
+## Version v2.1.3
+2022-05-16
+* DEV-13981 - Bug: CLI - dataset query clone stuck at 50% then "Killed"
+## Version v2.1.4
+2022-05-22
+* DEV-14182 - Bug: Cli - hide 'data upload' command
+## Version v2.1.5
+2022-07-31
+* DEV-14244 - Bug: CLI - "failed to upload ongoing stats" due to NaN in float
+* DEV-14633 - Bug: End sync did not complete, causing the experiment to get stuck in "terminating"
+## Version v2.1.6
+2022-08-09
+* DEV-14682 - Bug: git-Walki: CLI/SDK experiments goes into debug mode for Github+SSH integrated projects

data/lib/cnvrg/cli.rb

@@ -1008,6 +1008,7 @@ module Cnvrg
               abs_path = dataset_home + "/" + relative_path_dir
               abs_path = dataset_home if flatten
               fullpath = abs_path + "/" + file_name
+              fullpath = fullpath.gsub("//", "/")
 
               begin
                 FileUtils.mkdir_p(abs_path) unless File.exist? (fullpath)
@@ -1018,14 +1019,14 @@ module Cnvrg
               begin
                 unless File.exist?(fullpath)
                   downloader.safe_operation("#{abs_path}/#{file_name}") do
-                    File.open(fullpath, "w") { |file| file.write open(f["url"]).read }
+                    download = open(f["url"])
+                    IO.copy_stream(download, fullpath)
                   end
                 end
               rescue => e
                 log_message("Could not download file: #{f["fullpath"]}", Thor::Shell::Color::RED)
                 exit(1)
               end
-
             end
             #@executer.set_dataset_status(dataset: dataset.slug, status: "cloned") if @executer.present?
           rescue Interrupt
@@ -1847,7 +1848,7 @@ module Cnvrg
       log_start(__method__, args, options)
       project_home = Dir.pwd
       soft = options["soft"] || false
-      Project.stop_if_project_present(project_home, slug) if soft
+      Project.stop_if_project_present(project_home, slug, owner) if soft
       clone_resp = Project.clone_dir_remote(slug, owner, slug,true)
       exit 1 if not clone_resp
       idx_status = Project.new(get_project_home).generate_idx(files:[])
@@ -1917,7 +1918,7 @@ module Cnvrg
         clone_resp = false
         project_home = Dir.pwd
 
-        Project.stop_if_project_present(project_home, project_name) if soft
+        Project.stop_if_project_present(project_home, project_name, owner) if soft
 
         if remote and !git
           clone_resp = Project.clone_dir_remote(slug, owner, project_name,git)
@@ -2061,6 +2062,8 @@ module Cnvrg
     method_option :init, :type => :boolean, :aliases => ["--initial"], :desc => "initial sync", :default => false
     method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
     def sync_data_new(new_branch, force, verbose, commit, all_files, tags ,parallel, chunk_size, init, message)
+      log_message("This method is deprecated, please use 'data put' instead. for more info visit our docs: https://app.cnvrg.io/docs/cli/install.html#upload-files-to-a-dataset", Thor::Shell::Color::BLUE, !options["verbose"])
+      return
       verify_logged_in(true)
       log_start(__method__, args, options)
       log_message('Syncing dataset', Thor::Shell::Color::BLUE, !options["verbose"])
@@ -2085,6 +2088,8 @@ module Cnvrg
     method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
 
     def upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:nil, total_deleted: 0, total_downloaded: 0)
+      log_message("This method is deprecated, please use 'data put' instead. for more info visit our docs: https://app.cnvrg.io/docs/cli/install.html#upload-files-to-a-dataset", Thor::Shell::Color::BLUE, !options["verbose"])
+      return
       begin
         commit, files_list = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size, :message => message
         files_to_upload, upload_errors = invoke :upload_data_files,[commit, files_list: files_list],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
@@ -3416,9 +3421,6 @@ module Cnvrg
               end
 
               end_commit = @project.last_local_commit
-              if end_commit.present?
-                @exp.job_log(["Experiment end commit: #{end_commit}"])
-              end
 
                 # log_thread.join
               stats_thread.join if docker_stats
@@ -4731,8 +4733,14 @@ module Cnvrg
                 end
               end
             else
-              timestamp, value = data_result&.first&.dig('value')
-              stat_value = value.present? ? ("%.2f" % value) : 0 # converting 34.685929244444445 to 34.69
+              begin
+                timestamp, value = data_result&.first&.dig('value')
+                stat_value = value.present? ? ("%.2f" % value) : 0 # converting 34.685929244444445 to 34.69
+              rescue => e
+                Cnvrg::Logger.log_info("Failed converting string into float with error: #{e.message}")
+                Cnvrg::Logger.log_error(e)
+                stat_value = 0
+              end
               stat_value = stat_value.to_i == stat_value.to_f ? stat_value.to_i : stat_value.to_f # converting 34.00 to 34
               if query_name.include? 'block'
                 stats['block_io'] = {} if stats['block_io'].blank?
@@ -5013,10 +5021,10 @@ module Cnvrg
             else
               log_message("#{exp_name} is running should get logs", Thor::Shell::Color::BLUE)
               success, num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
-              if !success and exp["last_successful_commit"].present? and !copied_commits.include?(exp["last_successful_commit"])
+              if !success and exp["last_successful_commit"].present? and !copied_commits.include?(exp["last_successful_commit"]["sha1"])
                 log_message("Failed to get kube files, using last commit", Thor::Shell::Color::BLUE)
-                num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"])
-                copied_commits << exp["last_successful_commit"]
+                num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"]["sha1"])
+                copied_commits << exp["last_successful_commit"]["sha1"]
               end
             end
 

data/lib/cnvrg/data.rb

@@ -81,7 +81,7 @@ module Cnvrg
       end
     end
 
-    desc "data upload", "Upload files from local dataset directory to remote server"
+    desc "data upload", "Upload files from local dataset directory to remote server", :hide => true
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
@@ -101,7 +101,7 @@ module Cnvrg
       message = options["message"]
       cli.upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:message)
     end
-    desc 'data sync', 'Synchronise local dataset directory with remote server'
+    desc 'data sync', 'Synchronise local dataset directory with remote server', :hide => true
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false

data/lib/cnvrg/datafiles.rb

@@ -1393,13 +1393,13 @@ module Cnvrg
             in_threads: threads,
             isolation: true
         }
+
         Parallel.map(files["keys"], parallel_options) do |f|
           begin
             file_path = f['name']
             file_path = File.basename(f['name']) if flatten
             local_path = @dataset.local_path + '/' + file_path
             Cnvrg::Logger.log_info("Downloading #{local_path}")
-            progressbar.progress += 1 if progressbar.present?
             if local_path.end_with? "/"
               @downloader.mkdir(local_path, recursive: true)
               next
@@ -1420,6 +1420,7 @@ module Cnvrg
             end
 
             resp = @downloader.safe_download(storage_path, local_path)
+            progressbar.progress += 1 if progressbar.present?
             Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
           rescue => e
             Cnvrg::Logger.log_error(e)

data/lib/cnvrg/downloader/clients/azure_client.rb

@@ -1,4 +1,6 @@
+require 'open-uri'
 require 'azure/storage/blob'
+require 'azure/storage/common/core'
 
 module Cnvrg
   module Downloader
@@ -13,10 +15,27 @@ module Cnvrg
 
         def download(storage_path, local_path, decrypt: true)
           prepare_download(local_path)
+
           storage_path = Cnvrg::Helpers.decrypt(@key, @iv, storage_path) if decrypt
-          blob, content = client.get_blob(@container, storage_path)
-          ::File.open(local_path, 'wb') {|f| f.write(content)}
-          blob
+
+          # We generate a temp uri in order to stream the file instead of using "get_blob" that overflows memory
+          uri = client.send(:blob_uri, @container, storage_path)
+
+          generator = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(@account_name, @access_key)
+
+          expiring_url = generator.signed_uri(
+            uri,
+            false,
+            service: 'b',
+            resource: 'b',
+            permissions: 'r',
+            start: (Time.now - (5 * 60)).utc.iso8601, # start 5 minutes ago
+            expiry: (Time.now + 60 * 60 * 2).utc.iso8601 # expire in 2 hours
+          )
+
+          # Stream the file without loading it all into memory
+          download = open(expiring_url)
+          IO.copy_stream(download, local_path)
         end
 
         def upload(storage_path, local_path)

data/lib/cnvrg/helpers/agent.rb

@@ -59,7 +59,6 @@ class Cnvrg::Helpers::Agent
         not File.exists? file
       end
       return true if file_doesnt_exists.blank?
-      log_internal("Can't find file #{file_doesnt_exists}, stopping the job")
       return false
     end
     true
@@ -180,11 +179,17 @@ class Cnvrg::Helpers::Agent
       end
     end
     @exit_status = $?.exitstatus
+  rescue NoMethodError => e
+    log_internal("No Method Error: #{e}", level: LogLevel::ERROR)
+    @exit_status = 129
   rescue Timeout::Error
     Process.kill(0, @pid)
     @errors << {log: "Command timed out!", timestamp: Time.now}
     log_internal("Command timed out!", level: LogLevel::ERROR)
     @exit_status = 124
+  rescue => e
+    log_internal("Error: #{e}", level: LogLevel::ERROR)
+    @exit_status = 129
   ensure
     retry_command if @retries != 0 and @exit_status !=0
     @exit_status

data/lib/cnvrg/helpers/executer.rb

@@ -89,7 +89,7 @@ class Cnvrg::Helpers::Executer
       while agent_id.blank? or main_id.blank?
         grep_by = @job_id
         grep_by = "$(hostname)" if ENV['KUBERNETES_PORT'].present?
-        cntrs = `docker ps --format "table {{.ID}},{{.Names}}" | grep -i #{grep_by}`.split("\n").map{|x| x.strip}
+        cntrs = `docker ps --format "table {{.ID}},{{.Names}}" 2> /dev/null | grep -i #{grep_by}`.split("\n").map{|x| x.strip}
         agent_id = cntrs.find{|container_name| container_name.include? "agent"}.split(",").first rescue nil
         main_id = cntrs.find{|container_name| container_name.include? @main_name}.split(",").first rescue nil
         sleep(2)
@@ -168,6 +168,9 @@ class Cnvrg::Helpers::Executer
     while !success and retries < 100
       begin
         resp = Cnvrg::API.request(activity_url, "PUT", {stats: executer_stats})
+        if !resp
+          raise StandardError.new("Failed to send request to server")
+        end
         machine_activity = resp["machine_activity"]
         success = true
         puts("Connected to server")
@@ -216,6 +219,7 @@ class Cnvrg::Helpers::Executer
   def wait_for_main
     copy_file_to_main
     start_tiny_if_missing
+    retries = 0
     puts("Waiting for main container")
     STDOUT.flush
     got_response = false
@@ -233,9 +237,12 @@ class Cnvrg::Helpers::Executer
           got_response = true
         end
       rescue => e
-        puts("Failed to connect to main")
-        puts(e)
-        STDOUT.flush
+        retries += 1
+        if retries > 3
+          puts("Failed to connect to main")
+          puts(e.message)
+          STDOUT.flush
+        end
         sleep(0.1)
         next
       end
@@ -265,13 +272,30 @@ class Cnvrg::Helpers::Executer
   end
 
   def execute_cmds
-    pids = []
+    pids_by_slug = {}
     while true
       if @commands_q.empty?
         sleep(5)
         next
       end
       cmd = @commands_q.pop.symbolize_keys
+
+      if cmd[:wait_slug].present?
+        if pids_by_slug[cmd[:wait_slug]].present?
+          other_pid = pids_by_slug[cmd[:wait_slug]]
+          begin
+            Process.waitpid(other_pid, Process::WNOHANG)
+            running = true
+          rescue Errno::ECHILD => e
+            running = false
+          end
+          if running
+            @commands_q.push(cmd)
+            sleep(5)
+            next
+          end
+        end
+      end
       command_json = Cnvrg::API.request([activity_url, "commands", cmd[:slug]].join('/'), "GET")
 
       cmd_status = command_json["status"] rescue ""
@@ -288,10 +312,9 @@ class Cnvrg::Helpers::Executer
       else
         Process.detach(pid)
       end
-      pids << pid
+      pids_by_slug[cmd[:slug]] = pid
       ######
     end
-    pids
   end
 
   def merge_log_block(logs)
@@ -303,7 +326,7 @@ class Cnvrg::Helpers::Executer
     pod_name = `hostname`.strip rescue nil
     node_name = nil
     if pod_name.present?
-      pod_describe = `kubectl get pod #{pod_name} -o json` rescue nil
+      pod_describe = `kubectl get pod #{pod_name} -o json 2> /dev/null` rescue nil
       pod_describe = JSON.parse(pod_describe) rescue {}
       node_name = pod_describe["spec"]["nodeName"] rescue nil
     end
@@ -366,4 +389,4 @@ class Cnvrg::Helpers::Executer
     conn.options.open_timeout = open_timeout
     conn
   end
-end
+end

data/lib/cnvrg/project.rb

@@ -829,12 +829,12 @@ module Cnvrg
       Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/set_started", "POST", {job_type: job_type, job_id: job_id})
     end
 
-    def self.stop_if_project_present(project_home, project_name)
+    def self.stop_if_project_present(project_home, project_name, owner)
       cli = Cnvrg::CLI.new()
       config = YAML.load_file(project_home + "/.cnvrg/config.yml")
       local_commit = YAML.load_file(project_home + "/.cnvrg/idx.yml")[:commit] rescue nil
       return if local_commit.blank?
-      if config[:project_name] == project_name
+      if config[:project_name] == project_name && config[:owner] == owner
         cli.log_message("Project already present, clone aborted")
         exit(0)
       end

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '2.0.20'
+  VERSION = '2.1.6'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 2.0.20
+  version: 2.1.6
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-03-02 00:00:00.000000000 Z
+date: 2022-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler