cnvrg 1.11.28 → 2.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e617f50977eafa031b94bed3ba18753ecb4529924f74182525a5c6a53410fc5f
-  data.tar.gz: 40b755a41a73611dbc8f48802352ca73cac613c2e6d3fa70bf823e9e24f7c3f5
+  metadata.gz: b238ce877c0fa8da80c63c6d3d165ce974c250ea9c3f3d21ff092098356aa5a5
+  data.tar.gz: 7c7b4aa637b3bcaf5f18a7eb973d6995f0f1e25147a16a2ccfb1ce3d6dea91e3
 SHA512:
-  metadata.gz: 51abb620c15a66be237a4392ec7fad73e1a5f865f3932934b938028ee0b54558772b8dcc292c77379bbbe273c1771f2b3e85d250d492cafa0b0bc681cb7a52dc
-  data.tar.gz: 06fe73c3d4246c41b43923736c2ece0680a4f04522a317acdc576b6191d5ca74057f55bc6c5120456ac8ef0c5cfe7f328317655799b0d3d3c910db1cde871ad7
+  metadata.gz: fb1b724199e92fddfa2be4c04b9eb1a8a94e1df65759482fb6c3173f7c3c7632d236c22836749e5f1d6578115555b838285b58d1a95a42ce2df67d3a354da043
+  data.tar.gz: c840826e01dcd8061fdb1bf065f416d97ca3fe35575649171814741c32332b458c7bac57fa248e9afd45f337b60b18003077cc72d5927527490689572608ad6b

data/Readme.md

@@ -0,0 +1,42 @@
+
+## Version v1.11.15
+2021-03-30
+* DEV-208 - Task: Make sure the index name is constant over days
+* DEV-7555 - Bug: CLI: Error message is not correct when run a flow after removing the permission from Template.
+* DEV-7800 - New Feature: FR - add stdout to CLI logs (for logging in kibana etc.)
+* DEV-7928 - Bug: CLI - cnvrg clone doesnt show log message when files not found
+* DEV-7956 - Bug: CLI crashes from progressbar
+* DEV-8006 - Bug: Cli- cnvrg data put ,slash in the end url path will cause unique index error
+* DEV-8007 - Bug: Cli- Cnvrg data clone failed sometimes to load sts, there for clone crashed
+* DEV-8159 - New Feature: Oauth Proxy
+* DEV-8179 - New Feature: Add auto cache and link files in cache clone
+* DEV-8208 - Bug: Cli - cnvrg data put fails
+* DEV-8284 - Improvement: Use server instead of docker for agent communication
+* DEV-8434 - Bug: Rerun of experiment in git project doesn't show artifacts
+* DEV-8539 - Bug: SDK - in windows: e.sync doesnt perform sync
+* DEV-8621 - Improvement: Add more metrics 
+## Version v1.11.30
+2021-04-06
+## Version v1.11.31
+2021-04-22
+## Version v1.11.32
+2021-05-05
+* DEV-8868 - Bug: SDK - e.sync() in git project only creates empty "output" folder in commit
+## Version v2.0.1
+2021-06-13
+## Version v2.0.2
+2021-06-16
+* DEV-9694 - Bug: Download artifacts fails on authorization error 
+## Version v2.0.3
+2021-06-29
+* DEV-9919 - Bug: clone artifacts fails on "Not Authorize, Are you logged in?"
+## Version v2.0.4
+2021-07-08
+* DEV-9935 - Bug: CLI - cnvrg sync creates new commit but no blob versions
+## Version v2.0.5
+2021-07-11
+* DEV-10171 - Bug: experiment randomly fails with error- "Couldn't clone artifacts"
+* DEV-10189 - Bug: CLI Sync -file/folder with broken symlink will cause sync to fail
+## Version v2.0.6
+2021-07-18
+* DEV-10209 - Bug: some experiments in grid failed on cnvrg-cli commands (docker container id was missing)

data/cnvrg.gemspec

@@ -14,8 +14,8 @@ Gem::Specification.new do |spec|
 
     #spec.files         = `git ls-files`.split($/)
     spec.files = %w[cnvrg.gemspec] + Dir['*.md', 'bin/*', 'lib/**/*.rb']
-    spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-    spec.executables   = ['cnvrg']
+    spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+    spec.executables = ['cnvrg']
     spec.require_paths = ['lib']
 
     spec.add_development_dependency 'bundler'
@@ -23,14 +23,15 @@ Gem::Specification.new do |spec|
     spec.add_development_dependency 'rspec', '~> 3.0'
     spec.add_development_dependency 'vcr', '~> 3.0'
     spec.add_development_dependency 'aruba'
-    spec.add_development_dependency 'pry'	
-    
-    spec.add_runtime_dependency 'mimemagic', '~> 0.3.1','>=0.3.2'
+    spec.add_development_dependency 'pry'
+
+    spec.add_runtime_dependency 'ffi', '~> 1.9', '>= 1.9.10'
+    spec.add_runtime_dependency 'mimemagic', '~> 0.3.1', '>=0.3.7'
     spec.add_runtime_dependency 'faraday', '~> 0.15.2'
     spec.add_runtime_dependency 'netrc', '~> 0.11.0'
     spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
     spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
-    spec.add_runtime_dependency 'thor', '~> 0.19.0','>=0.19.1'
+    spec.add_runtime_dependency 'thor', '~> 0.19.0', '>=0.19.1'
     spec.add_runtime_dependency 'aws-sdk-s3', '~> 1'
     spec.add_runtime_dependency 'signet', '~> 0.11.0'
     spec.add_runtime_dependency 'google-cloud-env', '~> 1.2.1'
@@ -38,6 +39,7 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'google-cloud-storage', '~> 1.21.1'
     spec.add_runtime_dependency 'sucker_punch', '~> 2.0'
     spec.add_runtime_dependency 'urlcrypt', '~> 0.1.1'
+    spec.add_runtime_dependency 'filewatch', '~> 0.9.0'
     spec.add_runtime_dependency 'parallel', '~> 1.12.0'
     spec.add_runtime_dependency 'azure-storage-blob', '~> 1.1.0'
     spec.add_runtime_dependency 'logstash-logger', '~> 0.22.1'

data/lib/cnvrg/api.rb

@@ -72,11 +72,11 @@ module Cnvrg
                     if response.to_hash[:status].to_i != 200
                       Cnvrg::Logger.log_info("Got back bad status #{response.to_hash[:status]}")
                     end
-                    if [503, 502, 429].include?(response.to_hash[:status].to_i)
+                    if [503, 502, 429, 401].include?(response.to_hash[:status].to_i)
                       Cnvrg::Logger.log_info("Got back status #{response.to_hash[:status]}, will retry in #{5 * retries} seconds")
                       success = false
                       sleep(5 * retries)
-                      retries +=1
+                      retries += 1
                       next
                     end
                   rescue => e
@@ -112,11 +112,11 @@ module Cnvrg
                         if response.to_hash[:status].to_i != 200
                           Cnvrg::Logger.log_info("Got back bad status #{response.to_hash[:status]}")
                         end
-                        if [503, 502, 429].include?(response.to_hash[:status].to_i)
+                        if [503, 502, 429, 401].include?(response.to_hash[:status].to_i)
                           Cnvrg::Logger.log_info("Got back status #{response.to_hash[:status]}, will retry in #{5 * retries} seconds")
                           success = false
                           sleep(5 * retries)
-                          retries +=1
+                          retries += 1
                           next
                         end
                     rescue => e

data/lib/cnvrg/cli.rb

@@ -173,7 +173,7 @@ module Cnvrg
     desc "data [COMMAND]", "Upload and manage datasets", :hide => false
     subcommand "data", Data
 
-    desc "job", "manage running jobs", :hide => false
+    desc "job", "manage running jobs", :hide => true
     subcommand "job", JobCli
 
     desc "ssh", "ssh into running jobs", :hide => false
@@ -415,7 +415,7 @@ module Cnvrg
       end
     end
 
-    desc 'set_compression_path', 'Set compression path'
+    desc 'set_compression_path', 'Set compression path', :hide => true
     method_option :reset, :type => :boolean, :aliases => ["-r", "--reset"], :default => false
 
     def set_compression_path(*compression_path)
@@ -2311,6 +2311,7 @@ module Cnvrg
     method_option :suppress_exceptions, :type => :boolean, :aliases => ["--suppress-exceptions"], :default => true
     method_option :debug_mode, :type => :boolean, :aliases => ["--debug-mode"], :default => false
     method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 1000
+    method_option :local, :type => :boolean, :aliases => ["--local"], :default => true
 
     def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false, output_dir = "output", job_type = nil, job_slug = nil, suppress_exceptions = true,chunk_size=1000)
       begin
@@ -2328,6 +2329,8 @@ module Cnvrg
           exp_obj = nil
         end
 
+        local = options["local"]
+
         commit_msg = options["message"]
         if commit_msg.nil? or commit_msg.empty?
           commit_msg = ""
@@ -2349,7 +2352,7 @@ module Cnvrg
             if git_output_dir.ends_with? "/"
               git_output_dir = git_output_dir[0..-2]
             end
-            list = @project.generate_output_dir(git_output_dir)
+            list = @project.generate_output_dir(git_output_dir, local: local)
           end
           list += @project.generate_git_diff if options["git_diff"]
           spec_files_to_upload = list
@@ -2668,7 +2671,7 @@ module Cnvrg
       end
     end
 
-    desc 'commit before termination', 'Commit job code before termination'
+    desc 'commit before termination', 'Commit job code before termination', :hide => true
     def commit_before_termination()
       job_type = ENV['CNVRG_JOB_TYPE']
       job_id =  ENV['CNVRG_JOB_ID']
@@ -2678,7 +2681,7 @@ module Cnvrg
       log_error(e)
     end
 
-    desc 'update_job_commit', 'Update job with its last commit'
+    desc 'update_job_commit', 'Update job with its last commit' , :hide => true
     def update_job_commit()
       job_type = ENV['CNVRG_JOB_TYPE']
       job_id =  ENV['CNVRG_JOB_ID']
@@ -2868,7 +2871,7 @@ module Cnvrg
 
 
 
-    desc 'jump', 'Jump to specific commit'
+    desc 'jump COMMIT_ID', 'Jump to specific commit'
     def jump(commit_sha1)
       begin
         verify_logged_in()
@@ -3003,11 +3006,12 @@ module Cnvrg
     method_option :job_slug, :type => :string, :aliases => ["-j", "--job"], :default => nil
     method_option :job_type, :type => :string, :aliases => ["-jt", "--job_type"], :default => nil
     method_option :files, :type => :string, :aliases => ["--files"], :default => nil
-    method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
+    method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => 'output'
     method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
     method_option :suppress_exceptions, :type => :boolean, :aliases => ["--suppress-exceptions"], :default => true
     method_option :debug_mode, :type => :boolean, :aliases => ["--debug-mode"], :default => false
     method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 1000
+    method_option :local, :type => :boolean, :aliases => ["--local"], :default => true
 
     def sync(direct = true)
       verify_logged_in(true) if direct
@@ -3030,10 +3034,10 @@ module Cnvrg
       if run_download or options['debug_mode']
         invoke :download, [true, "", in_exp ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
       end
-      invoke :upload, [false, true,  direct, "",in_exp,options[:force], output_dir, job_type, job_slug ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
+      invoke :upload, [false, true, direct, "", in_exp, options[:force], output_dir, job_type, job_slug], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
              :ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"],
              :files => options["files"], :output_dir => output_dir, :job_slug => job_slug, :job_type => job_type, :suppress_exceptions => options["suppress_exceptions"],
-             :debug_mode => options['debug_mode'], :git_diff => options["git_diff"], :chunk_size => options["chunk_size"]
+             :debug_mode => options['debug_mode'], :git_diff => options["git_diff"], :chunk_size => options["chunk_size"], :local => options["local"]
 
     end
 
@@ -3199,6 +3203,7 @@ module Cnvrg
     method_option :output_dir, :type => :string, :aliases => ["-o", "--output_dir"], :default => nil
     method_option :data_query, :type => :string, :aliases => ["-q", "--query"], :default => nil
     method_option :use_bash, :type => :boolean, :aliases => ["-b", "--use_bash"], :default => false
+    method_option :docker_stats, :type => :boolean, :aliases => ["--docker_stats"], :default => true
 
     def exec(*cmd)
       log = []
@@ -3222,6 +3227,7 @@ module Cnvrg
       output_dir = options['output_dir'] || "output"
       project_home = get_project_home
       data_query = options["data_query"]
+      docker_stats = options["docker_stats"]
       @project = Project.new(project_home)
       if @project.is_git
         sync_before = false
@@ -3294,80 +3300,67 @@ module Cnvrg
             stdout, stderr = '', ''
             begin
               process_running = true
-              stats_thread = Thread.new do
-                while process_running do
-                  sleep 30
-                  begin
-                    stats = remote ? usage_metrics_in_docker(docker_id) : Helpers.ubuntu? ? {memory: memory_usage, cpu: cpu_usage} : {}
-                    if is_on_gpu
-                      gu = gpu_util(take_from_docker: options["gpu_util_from_docker"], docker_id: docker_id)
-                      stats['gpu_util'] = gu[0]
-                      stats['gpu'] = gu[1]
+              if docker_stats
+                stats_thread = Thread.new do
+                  while process_running do
+                    sleep 30
+                    begin
+                      stats = remote ? usage_metrics_in_docker(docker_id) : Helpers.ubuntu? ? { memory: memory_usage, cpu: cpu_usage } : {}
+                      if is_on_gpu
+                        gu = gpu_util(take_from_docker: options["gpu_util_from_docker"], docker_id: docker_id)
+                        stats['gpu_util'] = gu[0]
+                        stats['gpu'] = gu[1]
+                      end
+                      @exp.send_machine_stats [stats] unless stats.empty?
+                    rescue => e
+                      log_error(e)
+                      log_message("Failed to upload ongoing stats, continuing with experiment", Thor::Shell::Color::YELLOW)
                     end
-                    @exp.send_machine_stats [stats] unless stats.empty?
-                  rescue => e
-                    log_error(e)
-                    log_message("Failed to upload ongoing stats, continuing with experiment", Thor::Shell::Color::YELLOW)
                   end
                 end
               end
               start_time = Time.now
-              shell_type = options["use_bash"] ? "bash -l" : "sh"
               if @exp.get_cmd.present?
                 cmd = @exp.get_cmd
-                if options["docker_id"].present? # Escape for docker exec
-                  cmd = cmd.gsub("\"", "\\\"")
-                end
               end
-              if options["docker_id"].present?
-                  cmd = "docker exec -it #{options["docker_id"]} #{shell_type} -c \"#{cmd}\""
+              command_slug = (0...18).map { (65 + rand(26)).chr }.join
+              result_file = "/conf/result-#{command_slug}"
+              data = {cmd: cmd, async: true, format: true, file_name: result_file, use_script: true, use_bash: options["use_bash"]}
+
+              conn = Cnvrg::Helpers::Executer.get_main_conn
+              response = conn.post('command', data.to_json)
+              if response.to_hash[:status].to_i != 200
+                exit_status = 129
+                raise StandardError.new("Cant send command to slave")
               end
-              PTY.spawn(@exp.as_env, cmd) do |stdout, stdin, pid, stderr|
+              t = FileWatch::Tail.new
+              filename = result_file
+              lines = []
+              t.tail(filename)
+              t.subscribe do |path, line|
                 begin
-                  stdout.each do |line|
-                    cur_time = Time.now
-                    real_time = Time.now - real
-                    cur_log = {time: cur_time,
-                               message: line,
-                               type: "stdout",
-                               real: real_time
-                    }
-                    if print_log
-                      puts({log: line, timestamp: Time.now, exp_logs: true}.to_json)
-                    end
+                  cur_log = JSON.parse(line)
+                  if cur_log["type"] == "endMessage"
+                    exit_status = cur_log["real"].to_i
+                    break
+                  else
+                    puts(cur_log.to_json)
+                    STDOUT.flush
+                    cur_log["time"] = Time.parse(cur_log["timestamp"])
+                    cur_log["message"] = cur_log["message"].to_s + "\r\n"
                     log << cur_log
-                    if log.size >= 10
-                      @exp.upload_temp_log(log) unless log.empty?
-                      log = []
-                      elsif (start_time + 15.seconds) <= Time.now
-                      @exp.upload_temp_log(log) unless log.empty?
-                      log = []
-                      start_time = Time.now
-                    end
                   end
-                  if stderr
-                    stderr.each do |err|
-                      log << {time: Time.now, message: err, type: "stderr"}
-                    end
-                  end
-                rescue Errno::EIO => e
-                  log_error(e)
-                  if !log.empty?
-                    temp_log = log
-                    @exp.upload_temp_log(temp_log) unless temp_log.empty?
-                    log -= temp_log
+                  if log.size >= 10
+                    @exp.upload_temp_log(log)
+                    log = []
+                  elsif (start_time + 15.seconds) <= Time.now
+                    @exp.upload_temp_log(log) unless log.empty?
+                    log = []
+                    start_time = Time.now
                   end
-                rescue Errno::ENOENT => e
-                  exp_success = false
-                  log_message("command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED)
-                  log_error(e)
                 rescue => e
-                  res = @exp.end(log, 1, start_commit, 0, 0)
-                  log_message("Error occurred,aborting", Thor::Shell::Color::RED)
                   log_error(e)
-                  exit(0)
                 end
-                ::Process.wait pid
               end
               end_time = Time.now
               process_running = false
@@ -3375,14 +3368,13 @@ module Cnvrg
               if !log.empty?
 
                 temp_log = log
-                  @exp.upload_temp_log(temp_log) unless temp_log.empty?
+                @exp.upload_temp_log(temp_log)
                 log -= temp_log
               end
 
               cpu_average = cpu_total.inject(0) {|sum, el| sum + el}.to_f / cpu_total.size
               memory_average = memory_total.inject(0) {|sum, el| sum + el}.to_f / memory_total.size
-              exit_status = $?.exitstatus
-              if $?.exitstatus != 0
+              if exit_status != 0
                 exp_success = false
               end
 
@@ -3405,7 +3397,7 @@ module Cnvrg
               end
 
                 # log_thread.join
-              stats_thread.join
+              stats_thread.join if docker_stats
 
                 res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average, end_time: end_time)
 
@@ -3425,8 +3417,7 @@ module Cnvrg
               log_message("Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED)
               if @exp
                 # log_thread.join
-                Thread.kill(stats_thread)
-                exit_status = $?.exitstatus
+                Thread.kill(stats_thread) if docker_stats
                 if exit_status.blank?
                   exit_status = "-1"
                 end
@@ -3439,8 +3430,6 @@ module Cnvrg
 
               exit(1)
             end
-
-
           end
 
         end
@@ -3449,7 +3438,7 @@ module Cnvrg
         end_commit = @project.last_local_commit
         process_running = false
         # log_thread.join
-        stats_thread.join
+        stats_thread.join if docker_stats
 
         res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
         if container
@@ -3685,7 +3674,7 @@ module Cnvrg
       end
     end
 
-    desc 'deploy', 'Deploys model to production'
+    desc 'deploy', 'Deploys model to production', :hide => true
     method_option :small, :type => :boolean, :aliases => ["-s", "--small"], :default => false
     method_option :medium, :type => :boolean, :aliases => ["-m", "--medium"], :default => false
     method_option :large, :type => :boolean, :aliases => ["-l", "--large"], :default => false
@@ -3774,7 +3763,7 @@ module Cnvrg
     method_option :dataset_only_tree, :type => :boolean, :aliases => [ "--dataset_only_tree"], :default => false
     method_option :data_query, :type => :string, :aliases => ["-q", "--query"], :default => ""
 
-    desc 'notebook', 'Starts a notebook session remotely or locally'
+    desc 'notebook', 'Starts a notebook session remotely or locally', :hide => true
 
     def notebook
       verify_logged_in(true)
@@ -3901,7 +3890,7 @@ module Cnvrg
       end
     end
 
-    desc 'remote_notebook', 'Run notebook server on remote server'
+    desc 'remote_notebook', 'Run notebook server on remote server', :hide => true
     method_option :machine_type, :type => :string, :default => ""
     method_option :notebook_type, :type => :string, :aliases => ["-n", "--notebook_type"], :default => ""
     method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
@@ -4260,7 +4249,7 @@ module Cnvrg
 
     end
 
-    desc 'notebook_stop', 'Starts a new notebook environment'
+    desc 'notebook_stop', 'Stop notebook', :hide => true
     method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
     method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false, :desc => "run on remote machine"
     method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
@@ -4640,6 +4629,108 @@ module Cnvrg
       end
     end
 
+    desc 'Collect and send job utilization', '', :hide => true
+    method_option :prometheus_url, :type => :string, :aliases => ["--prometheus_url"], :desc => "prometheus url to collect metrics from"
+    method_option :node_name, :type => :string, :aliases => ["--node_name"], :desc => "machie activity node name"
+    method_option :machine, :type => :boolean, :aliases => ["--machine"], :desc => "get machine_query or cluster_query"
+    method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :desc => "collect gpu metrics", :default => false
+    method_option :gaudi, :type => :boolean, :aliases => ["--gaudi"], :desc => "collect gaudi metrics", :default => false
+    method_option :wait, :type => :numeric, :aliases => ["--wait"], :desc => "to to wait between querying", :default => 30
+    method_option :prom_user, :type => :string, :aliases => ["--prom_user"], :desc => "prometheus username", :default => nil
+    method_option :prom_password, :type => :string, :aliases => ["--prom_password"], :desc => "prometheus password", :default => nil
+    method_option :name, :type => :string, :aliases => ["--name"], :desc => "pod name - used for master-workers jobs", :default => nil
+
+    def collect_metrics
+      @exp = Experiment.new(ENV['CNVRG_OWNER'], ENV['CNVRG_PROJECT'], job_id: ENV['CNVRG_JOB_ID'])
+      prometheus_url = options[:prometheus_url].ends_with?("/") ? options[:prometheus_url] : "#{options[:prometheus_url]}/"
+      prom_user = options[:prom_user]
+      prom_password = options[:prom_password]
+      name = options[:name]
+
+      translate_result = Cnvrg::API_V2.request(
+        "#{ENV['CNVRG_OWNER']}/resources/translate_metrics",
+        'GET',
+        { gpu: options[:gpu], gaudi: options[:gaudi] }
+      )
+
+      is_machine = options[:machine]
+      while true do
+        begin
+          stats = {}
+          translate_result.each do |query_name, metric|
+            if is_machine
+              metric_query = metric['machine_query'].presence || metric['query']
+              query_content = metric_query.gsub('#JOB_SLUG#', ENV['CNVRG_JOB_ID']).gsub('#NODE_NAME#', options[:node_name])
+            else
+              metric_query = metric['cluster_query'].presence || metric['query']
+              pod_name = `hostname`.strip
+              query_content = metric_query.gsub('#JOB_SLUG#', pod_name).gsub('#NODE_NAME#', options[:node_name])
+            end
+            if metric_query.blank? || query_content.blank?
+              next
+            end
+            uri = URI("#{prometheus_url}api/v1/query?query=#{query_content}")
+            http = Net::HTTP.new(uri.host, uri.port)
+            http.use_ssl = uri.scheme == "https"
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            req = Net::HTTP::Get.new uri.request_uri
+            if prom_user.present?
+              req.basic_auth(Base64.decode64(prom_user), Base64.decode64(prom_password))
+            end
+            resp = http.request(req)
+            begin
+              result = JSON.parse(resp.body)
+            rescue JSON::ParserError => e
+              log_error(e)
+              next
+            end
+            data_result = result&.dig('data', 'result')
+            next unless data_result
+
+            if data_result.size > 1
+              stats[query_name] = {} unless query_name.include? 'block'
+              data_result.each_with_index do |res, i|
+                timestamp, value = res["value"]
+                uuid = res["metric"]["UUID"].presence || i
+                stat_value = value.present? ? ("%.2f" % value) : 0 # converting 34.685929244444445 to 34.69
+                stat_value = stat_value.to_i == stat_value.to_f ? stat_value.to_i : stat_value.to_f # converting 34.00 to 34
+                if query_name.include? 'block'
+                  uuid = res["metric"]["interface"].presence || i
+                  uuid = "#{name}-#{uuid}" if name.present?
+                  stats['block_io'] = {} if stats['block_io'].blank?
+                  io_type = query_name.split('_')[1]
+                  stats['block_io'][io_type] = {} if stats['block_io'][io_type].blank?
+                  stats['block_io'][io_type].merge!({ uuid => stat_value })
+                else
+                  stats[query_name][uuid] = stat_value
+                end
+              end
+            else
+              timestamp, value = data_result&.first&.dig('value')
+              stat_value = value.present? ? ("%.2f" % value) : 0 # converting 34.685929244444445 to 34.69
+              stat_value = stat_value.to_i == stat_value.to_f ? stat_value.to_i : stat_value.to_f # converting 34.00 to 34
+              if query_name.include? 'block'
+                stats['block_io'] = {} if stats['block_io'].blank?
+                io_type = query_name.split('_')[1]
+                if name.present?
+                  stats['block_io'][io_type] = {} if stats['block_io'][io_type].blank?
+                  stats['block_io'][io_type].merge!({ name => stat_value })
+                else
+                  stats['block_io'].merge!({ io_type => stat_value })
+                end
+              else
+                stats[query_name] = name.present? ? { name => stat_value } : stat_value
+              end
+            end
+          end
+          @exp.send_machine_stats [stats] unless stats.empty?
+        rescue => e
+          log_error(e)
+          log_message("Failed to upload ongoing stats, continuing with experiment", Thor::Shell::Color::YELLOW)
+        end
+        sleep options[:wait]
+      end
+    end
 
     desc '', '', :hide => true
 
@@ -4672,7 +4763,7 @@ module Cnvrg
     end
 
 
-    desc '', ''
+    desc '', '', :hide => true
 
     def download_built_image(image_name, image_slug)
       begin