cnvrg 2.0.20 → 2.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d5e7aa6c49c6bbe9ce99dcc85f90825d67ef8a4d066c84dd5978da99afb116b
-  data.tar.gz: 81e5d10c09beddd7da049ac29d2bd71fc611f3c2ca5bd4b2e2431457573b683d
+  metadata.gz: aa682e9cf25c1d37b533721888ded60b3ebc397717fc904dcd7703a592dd8853
+  data.tar.gz: bba09312bac1e44ec8b64358dead4036e173832cb605e56803bc368b8eff6dbc
 SHA512:
-  metadata.gz: c338755e158c6e1c03dc84c16f3015c1801875d98e58ad788da47b5cf2374b06fa62445279ddd9585b9e2dd3a62bee381af968f1e5925c2b74a9eedf0652b17b
-  data.tar.gz: 674fe36d75e00c70919067c2539a85c3c56c6850852faf00781f8419a3a3a1dabb3d925be56316a9d20903a869300b761a893f01a0cea94eef6703f99c34d5a8
+  metadata.gz: d8144e807411faa6c507c037309abd5e4919b317253147b6bb1eec8eaab2614ad84d32823bfc9dde405f881cefe4be9d975dd2dc21e47392c8416ddd0ed6c253
+  data.tar.gz: 46aebd82a4901774d605f40c26aee7607a0a3c407ddf1c3cc4f91625f678e5f2f0bcb8e53c92b3c5812150a6b4b98f491e3c21adace85196d82314e35ef7646e

data/Readme.md

@@ -80,4 +80,12 @@
 * DEV-13271 - Bug: CLI - on upload folders in working dir containing .cnvrg, dir not uploading - dir is on .cnvrgignore
 ## Version v2.0.20
 2022-02-27
-* DEV-12288 - Bug: wrong error message when upload fails
+* DEV-12288 - Bug: wrong error message when upload fails
+## Version v2.1.1
+2022-05-01
+## Version v2.1.2
+2022-05-08
+* DEV-13815 - Bug: CLI - remove "cnvrg data sync" command
+## Version v2.1.3
+2022-05-16
+* DEV-13981 - Bug: CLI - dataset query clone stuck at 50% then "Killed"

data/lib/cnvrg/cli.rb

@@ -1008,6 +1008,7 @@ module Cnvrg
               abs_path = dataset_home + "/" + relative_path_dir
               abs_path = dataset_home if flatten
               fullpath = abs_path + "/" + file_name
+              fullpath = fullpath.gsub("//", "/")
 
               begin
                 FileUtils.mkdir_p(abs_path) unless File.exist? (fullpath)
@@ -1018,14 +1019,14 @@ module Cnvrg
               begin
                 unless File.exist?(fullpath)
                   downloader.safe_operation("#{abs_path}/#{file_name}") do
-                    File.open(fullpath, "w") { |file| file.write open(f["url"]).read }
+                    download = open(f["url"])
+                    IO.copy_stream(download, fullpath)
                   end
                 end
               rescue => e
                 log_message("Could not download file: #{f["fullpath"]}", Thor::Shell::Color::RED)
                 exit(1)
               end
-
             end
             #@executer.set_dataset_status(dataset: dataset.slug, status: "cloned") if @executer.present?
           rescue Interrupt
@@ -2061,6 +2062,8 @@ module Cnvrg
     method_option :init, :type => :boolean, :aliases => ["--initial"], :desc => "initial sync", :default => false
     method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
     def sync_data_new(new_branch, force, verbose, commit, all_files, tags ,parallel, chunk_size, init, message)
+      log_message("This method is deprecated, please use 'data put' instead. for more info visit our docs: https://app.cnvrg.io/docs/cli/install.html#upload-files-to-a-dataset", Thor::Shell::Color::BLUE, !options["verbose"])
+      return
       verify_logged_in(true)
       log_start(__method__, args, options)
       log_message('Syncing dataset', Thor::Shell::Color::BLUE, !options["verbose"])
@@ -3416,9 +3419,6 @@ module Cnvrg
               end
 
               end_commit = @project.last_local_commit
-              if end_commit.present?
-                @exp.job_log(["Experiment end commit: #{end_commit}"])
-              end
 
                 # log_thread.join
               stats_thread.join if docker_stats
@@ -5013,10 +5013,10 @@ module Cnvrg
             else
               log_message("#{exp_name} is running should get logs", Thor::Shell::Color::BLUE)
               success, num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
-              if !success and exp["last_successful_commit"].present? and !copied_commits.include?(exp["last_successful_commit"])
+              if !success and exp["last_successful_commit"].present? and !copied_commits.include?(exp["last_successful_commit"]["sha1"])
                 log_message("Failed to get kube files, using last commit", Thor::Shell::Color::BLUE)
-                num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"])
-                copied_commits << exp["last_successful_commit"]
+                num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"]["sha1"])
+                copied_commits << exp["last_successful_commit"]["sha1"]
               end
             end
 

data/lib/cnvrg/data.rb

@@ -101,7 +101,7 @@ module Cnvrg
       message = options["message"]
       cli.upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:message)
     end
-    desc 'data sync', 'Synchronise local dataset directory with remote server'
+    desc 'data sync', 'Synchronise local dataset directory with remote server', :hide => true
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false

data/lib/cnvrg/datafiles.rb

@@ -1393,13 +1393,13 @@ module Cnvrg
             in_threads: threads,
             isolation: true
         }
+
         Parallel.map(files["keys"], parallel_options) do |f|
           begin
             file_path = f['name']
             file_path = File.basename(f['name']) if flatten
             local_path = @dataset.local_path + '/' + file_path
             Cnvrg::Logger.log_info("Downloading #{local_path}")
-            progressbar.progress += 1 if progressbar.present?
             if local_path.end_with? "/"
               @downloader.mkdir(local_path, recursive: true)
               next
@@ -1420,6 +1420,7 @@ module Cnvrg
             end
 
             resp = @downloader.safe_download(storage_path, local_path)
+            progressbar.progress += 1 if progressbar.present?
             Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
           rescue => e
             Cnvrg::Logger.log_error(e)

data/lib/cnvrg/downloader/clients/azure_client.rb

@@ -1,4 +1,6 @@
+require 'open-uri'
 require 'azure/storage/blob'
+require 'azure/storage/common/core'
 
 module Cnvrg
   module Downloader
@@ -13,10 +15,27 @@ module Cnvrg
 
         def download(storage_path, local_path, decrypt: true)
           prepare_download(local_path)
+
           storage_path = Cnvrg::Helpers.decrypt(@key, @iv, storage_path) if decrypt
-          blob, content = client.get_blob(@container, storage_path)
-          ::File.open(local_path, 'wb') {|f| f.write(content)}
-          blob
+
+          # We generate a temp uri in order to stream the file instead of using "get_blob" that overflows memory
+          uri = client.send(:blob_uri, @container, storage_path)
+
+          generator = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(@account_name, @access_key)
+
+          expiring_url = generator.signed_uri(
+            uri,
+            false,
+            service: 'b',
+            resource: 'b',
+            permissions: 'r',
+            start: (Time.now - (5 * 60)).utc.iso8601, # start 5 minutes ago
+            expiry: (Time.now + 60 * 60 * 2).utc.iso8601 # expire in 2 hours
+          )
+
+          # Stream the file without loading it all into memory
+          download = open(expiring_url)
+          IO.copy_stream(download, local_path)
         end
 
         def upload(storage_path, local_path)

data/lib/cnvrg/helpers/agent.rb

@@ -59,7 +59,6 @@ class Cnvrg::Helpers::Agent
         not File.exists? file
       end
       return true if file_doesnt_exists.blank?
-      log_internal("Can't find file #{file_doesnt_exists}, stopping the job")
       return false
     end
     true

data/lib/cnvrg/helpers/executer.rb

@@ -89,7 +89,7 @@ class Cnvrg::Helpers::Executer
       while agent_id.blank? or main_id.blank?
         grep_by = @job_id
         grep_by = "$(hostname)" if ENV['KUBERNETES_PORT'].present?
-        cntrs = `docker ps --format "table {{.ID}},{{.Names}}" | grep -i #{grep_by}`.split("\n").map{|x| x.strip}
+        cntrs = `docker ps --format "table {{.ID}},{{.Names}}" 2> /dev/null | grep -i #{grep_by}`.split("\n").map{|x| x.strip}
         agent_id = cntrs.find{|container_name| container_name.include? "agent"}.split(",").first rescue nil
         main_id = cntrs.find{|container_name| container_name.include? @main_name}.split(",").first rescue nil
         sleep(2)
@@ -168,6 +168,9 @@ class Cnvrg::Helpers::Executer
     while !success and retries < 100
       begin
         resp = Cnvrg::API.request(activity_url, "PUT", {stats: executer_stats})
+        if !resp
+          raise StandardError.new("Failed to send request to server")
+        end
         machine_activity = resp["machine_activity"]
         success = true
         puts("Connected to server")
@@ -216,6 +219,7 @@ class Cnvrg::Helpers::Executer
   def wait_for_main
     copy_file_to_main
     start_tiny_if_missing
+    retries = 0
     puts("Waiting for main container")
     STDOUT.flush
     got_response = false
@@ -233,9 +237,12 @@ class Cnvrg::Helpers::Executer
           got_response = true
         end
       rescue => e
-        puts("Failed to connect to main")
-        puts(e)
-        STDOUT.flush
+        retries += 1
+        if retries > 3
+          puts("Failed to connect to main")
+          puts(e.message)
+          STDOUT.flush
+        end
         sleep(0.1)
         next
       end
@@ -265,13 +272,30 @@ class Cnvrg::Helpers::Executer
   end
 
   def execute_cmds
-    pids = []
+    pids_by_slug = {}
     while true
       if @commands_q.empty?
         sleep(5)
         next
       end
       cmd = @commands_q.pop.symbolize_keys
+
+      if cmd[:wait_slug].present?
+        if pids_by_slug[cmd[:wait_slug]].present?
+          other_pid = pids_by_slug[cmd[:wait_slug]]
+          begin
+            Process.waitpid(other_pid, Process::WNOHANG)
+            running = true
+          rescue Errno::ECHILD => e
+            running = false
+          end
+          if running
+            @commands_q.push(cmd)
+            sleep(5)
+            next
+          end
+        end
+      end
       command_json = Cnvrg::API.request([activity_url, "commands", cmd[:slug]].join('/'), "GET")
 
       cmd_status = command_json["status"] rescue ""
@@ -288,10 +312,9 @@ class Cnvrg::Helpers::Executer
       else
         Process.detach(pid)
       end
-      pids << pid
+      pids_by_slug[cmd[:slug]] = pid
       ######
     end
-    pids
   end
 
   def merge_log_block(logs)
@@ -303,7 +326,7 @@ class Cnvrg::Helpers::Executer
     pod_name = `hostname`.strip rescue nil
     node_name = nil
     if pod_name.present?
-      pod_describe = `kubectl get pod #{pod_name} -o json` rescue nil
+      pod_describe = `kubectl get pod #{pod_name} -o json 2> /dev/null` rescue nil
       pod_describe = JSON.parse(pod_describe) rescue {}
       node_name = pod_describe["spec"]["nodeName"] rescue nil
     end
@@ -366,4 +389,4 @@ class Cnvrg::Helpers::Executer
     conn.options.open_timeout = open_timeout
     conn
   end
-end
+end

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '2.0.20'
+  VERSION = '2.1.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 2.0.20
+  version: 2.1.3
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-03-02 00:00:00.000000000 Z
+date: 2022-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler