cnvrg 2.0.18 → 2.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 30e82404c79b7780d736c0f4996ab1173cbbbfbc13a68c58a0dc0364775c7d5d
-  data.tar.gz: 6a846acb27781403e27c39ffd36f2b2149b771307b49375356cabd8f769fafba
+  metadata.gz: ffe47307abd2feac46497f34ceec95fab9a866324a235db3ab6f0c61129a7a9d
+  data.tar.gz: cd8316866861c8b16ec4a4d1d001d8da20f79e34ab95ea38a850a8bce79d6d66
 SHA512:
-  metadata.gz: af34ebd4026b57c3111686f066f5c88cda3851ba27fe13a8273848274b614ba088d055ba1334c85842aea1142a113701e1de63ddf0ca5dc141c891762678dcac
-  data.tar.gz: 79fe82877378207d25a7ab4a7f73414fde0d90a5df128e4c1bd513a3f441822e5a67f9d8f04986b266ee111a66084875008a862926e285f94960cb2a90b49198
+  metadata.gz: 9b73987d1023e4aa2600ca11555078194398e7e10cb1d0314569aac8ab24a7385a41c4e86e70a169a4502f122e56baa1bde68ea5cb13706c666996c7a92b5244
+  data.tar.gz: 8dbd96c0b77e8c88d52254d04af38efa20fcab1c003027c8ee4b8e5918dc01ff35ae35505ec5d36687183925847438d5cb49eb8063a2db2ed2784a13e09900d3

data/Readme.md

@@ -74,4 +74,25 @@
 * DEV-10581 - Bug: CLI - getting 404 response in "cnvrg set_default_owner"
 ## Version v2.0.18
 2022-01-31
-* DEV-12637 - Bug: Dataset - creating file from CLI/SDK in a folder with + sign, replaces + with space AND creates 2 folders
+* DEV-12637 - Bug: Dataset - creating file from CLI/SDK in a folder with + sign, replaces + with space AND creates 2 folders
+## Version v2.0.19
+2022-02-22
+* DEV-13271 - Bug: CLI - on upload folders in working dir containing .cnvrg, dir not uploading - dir is on .cnvrgignore
+## Version v2.0.20
+2022-02-27
+* DEV-12288 - Bug: wrong error message when upload fails
+## Version v2.1.1
+2022-05-01
+## Version v2.1.2
+2022-05-08
+* DEV-13815 - Bug: CLI - remove "cnvrg data sync" command
+## Version v2.1.3
+2022-05-16
+* DEV-13981 - Bug: CLI - dataset query clone stuck at 50% then "Killed"
+## Version v2.1.4
+2022-05-22
+* DEV-14182 - Bug: Cli - hide 'data upload' command
+## Version v2.1.5
+2022-07-31
+* DEV-14244 - Bug: CLI - "failed to upload ongoing stats" due to NaN in float
+* DEV-14633 - Bug: End sync did not complete, causing the experiment to get stuck in "terminating"

data/lib/cnvrg/cli.rb

@@ -1008,6 +1008,7 @@ module Cnvrg
               abs_path = dataset_home + "/" + relative_path_dir
               abs_path = dataset_home if flatten
               fullpath = abs_path + "/" + file_name
+              fullpath = fullpath.gsub("//", "/")
 
               begin
                 FileUtils.mkdir_p(abs_path) unless File.exist? (fullpath)
@@ -1018,14 +1019,14 @@ module Cnvrg
               begin
                 unless File.exist?(fullpath)
                   downloader.safe_operation("#{abs_path}/#{file_name}") do
-                    File.open(fullpath, "w") { |file| file.write open(f["url"]).read }
+                    download = open(f["url"])
+                    IO.copy_stream(download, fullpath)
                   end
                 end
               rescue => e
                 log_message("Could not download file: #{f["fullpath"]}", Thor::Shell::Color::RED)
                 exit(1)
               end
-
             end
             #@executer.set_dataset_status(dataset: dataset.slug, status: "cloned") if @executer.present?
           rescue Interrupt
@@ -2061,6 +2062,8 @@ module Cnvrg
     method_option :init, :type => :boolean, :aliases => ["--initial"], :desc => "initial sync", :default => false
     method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
     def sync_data_new(new_branch, force, verbose, commit, all_files, tags ,parallel, chunk_size, init, message)
+      log_message("This method is deprecated, please use 'data put' instead. for more info visit our docs: https://app.cnvrg.io/docs/cli/install.html#upload-files-to-a-dataset", Thor::Shell::Color::BLUE, !options["verbose"])
+      return
       verify_logged_in(true)
       log_start(__method__, args, options)
       log_message('Syncing dataset', Thor::Shell::Color::BLUE, !options["verbose"])
@@ -2085,6 +2088,8 @@ module Cnvrg
     method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
 
     def upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:nil, total_deleted: 0, total_downloaded: 0)
+      log_message("This method is deprecated, please use 'data put' instead. for more info visit our docs: https://app.cnvrg.io/docs/cli/install.html#upload-files-to-a-dataset", Thor::Shell::Color::BLUE, !options["verbose"])
+      return
       begin
         commit, files_list = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size, :message => message
         files_to_upload, upload_errors = invoke :upload_data_files,[commit, files_list: files_list],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
@@ -2377,7 +2382,11 @@ module Cnvrg
         if ignore.nil? or ignore.empty?
           ignore = ignore_list
         end
-        data_ignore = data_dir_include()
+
+        if job_type != "Experiment"
+          data_ignore = data_dir_include()
+        end
+
         if !data_ignore.nil?
           if ignore.nil? or ignore.empty?
             ignore = data_ignore
@@ -3412,9 +3421,6 @@ module Cnvrg
               end
 
               end_commit = @project.last_local_commit
-              if end_commit.present?
-                @exp.job_log(["Experiment end commit: #{end_commit}"])
-              end
 
                 # log_thread.join
               stats_thread.join if docker_stats
@@ -4727,8 +4733,14 @@ module Cnvrg
                 end
               end
             else
-              timestamp, value = data_result&.first&.dig('value')
-              stat_value = value.present? ? ("%.2f" % value) : 0 # converting 34.685929244444445 to 34.69
+              begin
+                timestamp, value = data_result&.first&.dig('value')
+                stat_value = value.present? ? ("%.2f" % value) : 0 # converting 34.685929244444445 to 34.69
+              rescue => e
+                Cnvrg::Logger.log_info("Failed converting string into float with error: #{e.message}")
+                Cnvrg::Logger.log_error(e)
+                stat_value = 0
+              end
               stat_value = stat_value.to_i == stat_value.to_f ? stat_value.to_i : stat_value.to_f # converting 34.00 to 34
               if query_name.include? 'block'
                 stats['block_io'] = {} if stats['block_io'].blank?
@@ -5009,10 +5021,10 @@ module Cnvrg
             else
               log_message("#{exp_name} is running should get logs", Thor::Shell::Color::BLUE)
               success, num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
-              if !success and exp["last_successful_commit"].present? and !copied_commits.include?(exp["last_successful_commit"])
+              if !success and exp["last_successful_commit"].present? and !copied_commits.include?(exp["last_successful_commit"]["sha1"])
                 log_message("Failed to get kube files, using last commit", Thor::Shell::Color::BLUE)
-                num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"])
-                copied_commits << exp["last_successful_commit"]
+                num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"]["sha1"])
+                copied_commits << exp["last_successful_commit"]["sha1"]
               end
             end
 

data/lib/cnvrg/data.rb

@@ -81,7 +81,7 @@ module Cnvrg
       end
     end
 
-    desc "data upload", "Upload files from local dataset directory to remote server"
+    desc "data upload", "Upload files from local dataset directory to remote server", :hide => true
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
@@ -101,7 +101,7 @@ module Cnvrg
       message = options["message"]
       cli.upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:message)
     end
-    desc 'data sync', 'Synchronise local dataset directory with remote server'
+    desc 'data sync', 'Synchronise local dataset directory with remote server', :hide => true
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false

data/lib/cnvrg/datafiles.rb

@@ -1393,13 +1393,13 @@ module Cnvrg
             in_threads: threads,
             isolation: true
         }
+
         Parallel.map(files["keys"], parallel_options) do |f|
           begin
             file_path = f['name']
             file_path = File.basename(f['name']) if flatten
             local_path = @dataset.local_path + '/' + file_path
             Cnvrg::Logger.log_info("Downloading #{local_path}")
-            progressbar.progress += 1 if progressbar.present?
             if local_path.end_with? "/"
               @downloader.mkdir(local_path, recursive: true)
               next
@@ -1420,6 +1420,7 @@ module Cnvrg
             end
 
             resp = @downloader.safe_download(storage_path, local_path)
+            progressbar.progress += 1 if progressbar.present?
             Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
           rescue => e
             Cnvrg::Logger.log_error(e)

data/lib/cnvrg/downloader/clients/azure_client.rb

@@ -1,4 +1,6 @@
+require 'open-uri'
 require 'azure/storage/blob'
+require 'azure/storage/common/core'
 
 module Cnvrg
   module Downloader
@@ -13,10 +15,27 @@ module Cnvrg
 
         def download(storage_path, local_path, decrypt: true)
           prepare_download(local_path)
+
           storage_path = Cnvrg::Helpers.decrypt(@key, @iv, storage_path) if decrypt
-          blob, content = client.get_blob(@container, storage_path)
-          ::File.open(local_path, 'wb') {|f| f.write(content)}
-          blob
+
+          # We generate a temp uri in order to stream the file instead of using "get_blob" that overflows memory
+          uri = client.send(:blob_uri, @container, storage_path)
+
+          generator = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(@account_name, @access_key)
+
+          expiring_url = generator.signed_uri(
+            uri,
+            false,
+            service: 'b',
+            resource: 'b',
+            permissions: 'r',
+            start: (Time.now - (5 * 60)).utc.iso8601, # start 5 minutes ago
+            expiry: (Time.now + 60 * 60 * 2).utc.iso8601 # expire in 2 hours
+          )
+
+          # Stream the file without loading it all into memory
+          download = open(expiring_url)
+          IO.copy_stream(download, local_path)
         end
 
         def upload(storage_path, local_path)

data/lib/cnvrg/files.rb

@@ -106,7 +106,7 @@ module Cnvrg
             commit: commit_sha1
         })
         unless Cnvrg::CLI.is_response_success(resp, false)
-          raise StandardError.new("unsupported character: folder name can not include / \\  * : ? \" | ")
+          raise StandardError.new("Cant upload files to the server")
         end
         # resolve bucket
         res = resp['result']

data/lib/cnvrg/helpers/agent.rb

@@ -59,7 +59,6 @@ class Cnvrg::Helpers::Agent
         not File.exists? file
       end
       return true if file_doesnt_exists.blank?
-      log_internal("Can't find file #{file_doesnt_exists}, stopping the job")
       return false
     end
     true
@@ -180,11 +179,17 @@ class Cnvrg::Helpers::Agent
       end
     end
     @exit_status = $?.exitstatus
+  rescue NoMethodError => e
+    log_internal("No Method Error: #{e}", level: LogLevel::ERROR)
+    @exit_status = 129
   rescue Timeout::Error
     Process.kill(0, @pid)
     @errors << {log: "Command timed out!", timestamp: Time.now}
     log_internal("Command timed out!", level: LogLevel::ERROR)
     @exit_status = 124
+  rescue => e
+    log_internal("Error: #{e}", level: LogLevel::ERROR)
+    @exit_status = 129
   ensure
     retry_command if @retries != 0 and @exit_status !=0
     @exit_status

data/lib/cnvrg/helpers/executer.rb

@@ -89,7 +89,7 @@ class Cnvrg::Helpers::Executer
       while agent_id.blank? or main_id.blank?
         grep_by = @job_id
         grep_by = "$(hostname)" if ENV['KUBERNETES_PORT'].present?
-        cntrs = `docker ps --format "table {{.ID}},{{.Names}}" | grep -i #{grep_by}`.split("\n").map{|x| x.strip}
+        cntrs = `docker ps --format "table {{.ID}},{{.Names}}" 2> /dev/null | grep -i #{grep_by}`.split("\n").map{|x| x.strip}
         agent_id = cntrs.find{|container_name| container_name.include? "agent"}.split(",").first rescue nil
         main_id = cntrs.find{|container_name| container_name.include? @main_name}.split(",").first rescue nil
         sleep(2)
@@ -168,6 +168,9 @@ class Cnvrg::Helpers::Executer
     while !success and retries < 100
       begin
         resp = Cnvrg::API.request(activity_url, "PUT", {stats: executer_stats})
+        if !resp
+          raise StandardError.new("Failed to send request to server")
+        end
         machine_activity = resp["machine_activity"]
         success = true
         puts("Connected to server")
@@ -216,6 +219,7 @@ class Cnvrg::Helpers::Executer
   def wait_for_main
     copy_file_to_main
     start_tiny_if_missing
+    retries = 0
     puts("Waiting for main container")
     STDOUT.flush
     got_response = false
@@ -233,9 +237,12 @@ class Cnvrg::Helpers::Executer
           got_response = true
         end
       rescue => e
-        puts("Failed to connect to main")
-        puts(e)
-        STDOUT.flush
+        retries += 1
+        if retries > 3
+          puts("Failed to connect to main")
+          puts(e.message)
+          STDOUT.flush
+        end
         sleep(0.1)
         next
       end
@@ -265,13 +272,30 @@ class Cnvrg::Helpers::Executer
   end
 
   def execute_cmds
-    pids = []
+    pids_by_slug = {}
     while true
       if @commands_q.empty?
         sleep(5)
         next
       end
       cmd = @commands_q.pop.symbolize_keys
+
+      if cmd[:wait_slug].present?
+        if pids_by_slug[cmd[:wait_slug]].present?
+          other_pid = pids_by_slug[cmd[:wait_slug]]
+          begin
+            Process.waitpid(other_pid, Process::WNOHANG)
+            running = true
+          rescue Errno::ECHILD => e
+            running = false
+          end
+          if running
+            @commands_q.push(cmd)
+            sleep(5)
+            next
+          end
+        end
+      end
       command_json = Cnvrg::API.request([activity_url, "commands", cmd[:slug]].join('/'), "GET")
 
       cmd_status = command_json["status"] rescue ""
@@ -288,10 +312,9 @@ class Cnvrg::Helpers::Executer
       else
         Process.detach(pid)
       end
-      pids << pid
+      pids_by_slug[cmd[:slug]] = pid
       ######
     end
-    pids
   end
 
   def merge_log_block(logs)
@@ -303,7 +326,7 @@ class Cnvrg::Helpers::Executer
     pod_name = `hostname`.strip rescue nil
     node_name = nil
     if pod_name.present?
-      pod_describe = `kubectl get pod #{pod_name} -o json` rescue nil
+      pod_describe = `kubectl get pod #{pod_name} -o json 2> /dev/null` rescue nil
       pod_describe = JSON.parse(pod_describe) rescue {}
       node_name = pod_describe["spec"]["nodeName"] rescue nil
     end
@@ -366,4 +389,4 @@ class Cnvrg::Helpers::Executer
     conn.options.open_timeout = open_timeout
     conn
   end
-end
+end

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '2.0.18'
+  VERSION = '2.1.5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 2.0.18
+  version: 2.1.5
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-01-31 00:00:00.000000000 Z
+date: 2022-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler