cnvrg 1.11.25 → 1.11.31

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4fa2d41b1f7a80c14522b607d157762fccbee612e2b5663e41f102dde8ea8b50
-  data.tar.gz: 1d8acf0aaeb2e47c9d9bb424a1a7f8cba2e75cddb0dced119309aa49702f21d8
+  metadata.gz: a31d199ccd8981e455384ced40cc63a0a9a687982338cbc85e3a9413c2968a2c
+  data.tar.gz: 10d4da464f6ef7b507d563e2976cf17f5d6cd8bcfeab742c83ec5bfa1ab575e2
 SHA512:
-  metadata.gz: 86fa4e56ddb38cce2b6e18efdeab1316570a42bfe3f450c1eb497dba99a2444a10c2042c51ff3d5da3507c1ce22a2e7cedadd203e3514b93dd0c9c494dcab785
-  data.tar.gz: c3388652c80427fd5e8e3e89289c853f238a2a01eddd46a04263e9e6df44feb64f2c315307ccad055ba7dad6dc8dad0c6619183fb7adcc0cf1a67e36553717d5
+  metadata.gz: 2495cc0d77ada40ae2c6da54ada72710f28facba737b0a37c0dd2cb904d37d32dd23d2cba476fb3854ec47e19eaa11fb34913c2038d6ea059378ad0d460e905a
+  data.tar.gz: dbe50928a5d25259f78fc1dcdab474545b71308b7e1d798678fb2cd8786f0b0b2eda78062bcef6bcfdb16c9481bc47349c6f571de9cfe89f547c57f583bea403

data/Readme.md

@@ -0,0 +1,21 @@
+
+## Version v1.11.15
+2021-03-30
+* DEV-208 - Task: Make sure the index name is constant over days
+* DEV-7555 - Bug: CLI: Error message is not correct when run a flow after removing the permission from Template.
+* DEV-7800 - New Feature: FR - add stdout to CLI logs (for logging in kibana etc.)
+* DEV-7928 - Bug: CLI - cnvrg clone doesnt show log message when files not found
+* DEV-7956 - Bug: CLI crashes from progressbar
+* DEV-8006 - Bug: Cli- cnvrg data put ,slash in the end url path will cause unique index error
+* DEV-8007 - Bug: Cli- Cnvrg data clone failed sometimes to load sts, there for clone crashed
+* DEV-8159 - New Feature: Oauth Proxy
+* DEV-8179 - New Feature: Add auto cache and link files in cache clone
+* DEV-8208 - Bug: Cli - cnvrg data put fails
+* DEV-8284 - Improvement: Use server instead of docker for agent communication
+* DEV-8434 - Bug: Rerun of experiment in git project doesn't show artifacts
+* DEV-8539 - Bug: SDK - in windows: e.sync doesnt perform sync
+* DEV-8621 - Improvement: Add more metrics 
+## Version v1.11.30
+2021-04-06
+## Version v1.11.31
+2021-04-22

data/cnvrg.gemspec

@@ -6,7 +6,7 @@ require 'cnvrg/version'
 Gem::Specification.new do |spec|
     spec.name          = 'cnvrg'
     spec.version       = Cnvrg::VERSION
-    spec.authors       = ['Yochay Ettun', 'Leah Kolben', 'Omer Shacham']
+    spec.authors       = ['Yochay Ettun', 'Leah Kolben']
     spec.email         = ['info@cnvrg.io']
     spec.summary       = %q{A CLI tool for interacting with cnvrg.io.}
     spec.description   = %q{A CLI tool for interacting with cnvrg.io.}
@@ -14,8 +14,8 @@ Gem::Specification.new do |spec|
 
     #spec.files         = `git ls-files`.split($/)
     spec.files = %w[cnvrg.gemspec] + Dir['*.md', 'bin/*', 'lib/**/*.rb']
-    spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-    spec.executables   = ['cnvrg']
+    spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+    spec.executables = ['cnvrg']
     spec.require_paths = ['lib']
 
     spec.add_development_dependency 'bundler'
@@ -25,12 +25,13 @@ Gem::Specification.new do |spec|
     spec.add_development_dependency 'aruba'
     spec.add_development_dependency 'pry'
 
-    spec.add_runtime_dependency 'mimemagic', '~> 0.3.1','>=0.3.2'
+    spec.add_runtime_dependency 'ffi', '~> 1.9', '>= 1.9.10'
+    spec.add_runtime_dependency 'mimemagic', '~> 0.3.1', '>=0.3.7'
     spec.add_runtime_dependency 'faraday', '~> 0.15.2'
     spec.add_runtime_dependency 'netrc', '~> 0.11.0'
     spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
     spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
-    spec.add_runtime_dependency 'thor', '~> 0.19.0','>=0.19.1'
+    spec.add_runtime_dependency 'thor', '~> 0.19.0', '>=0.19.1'
     spec.add_runtime_dependency 'aws-sdk-s3', '~> 1'
     spec.add_runtime_dependency 'signet', '~> 0.11.0'
     spec.add_runtime_dependency 'google-cloud-env', '~> 1.2.1'

data/lib/cnvrg/cli.rb

@@ -2320,7 +2320,6 @@ module Cnvrg
         @project = Project.new(get_project_home)
         chunk_size = chunk_size ? chunk_size : options["chunk_size"]
 
-
         # Enable local/experiment exception logging
         suppress_exceptions = suppress_exceptions ? suppress_exceptions : options[:suppress_exceptions]
         if in_exp
@@ -3200,6 +3199,7 @@ module Cnvrg
     method_option :output_dir, :type => :string, :aliases => ["-o", "--output_dir"], :default => nil
     method_option :data_query, :type => :string, :aliases => ["-q", "--query"], :default => nil
     method_option :use_bash, :type => :boolean, :aliases => ["-b", "--use_bash"], :default => false
+    method_option :docker_stats, :type => :boolean, :aliases => ["--docker_stats"], :default => true
 
     def exec(*cmd)
       log = []
@@ -3223,6 +3223,7 @@ module Cnvrg
       output_dir = options['output_dir'] || "output"
       project_home = get_project_home
       data_query = options["data_query"]
+      docker_stats = options["docker_stats"]
       @project = Project.new(project_home)
       if @project.is_git
         sync_before = false
@@ -3295,20 +3296,22 @@ module Cnvrg
             stdout, stderr = '', ''
             begin
               process_running = true
-              stats_thread = Thread.new do
-                while process_running do
-                  sleep 30
-                  begin
-                    stats = remote ? usage_metrics_in_docker(docker_id) : Helpers.ubuntu? ? {memory: memory_usage, cpu: cpu_usage} : {}
-                    if is_on_gpu
-                      gu = gpu_util(take_from_docker: options["gpu_util_from_docker"], docker_id: docker_id)
-                      stats['gpu_util'] = gu[0]
-                      stats['gpu'] = gu[1]
+              if docker_stats
+                stats_thread = Thread.new do
+                  while process_running do
+                    sleep 30
+                    begin
+                      stats = remote ? usage_metrics_in_docker(docker_id) : Helpers.ubuntu? ? { memory: memory_usage, cpu: cpu_usage } : {}
+                      if is_on_gpu
+                        gu = gpu_util(take_from_docker: options["gpu_util_from_docker"], docker_id: docker_id)
+                        stats['gpu_util'] = gu[0]
+                        stats['gpu'] = gu[1]
+                      end
+                      @exp.send_machine_stats [stats] unless stats.empty?
+                    rescue => e
+                      log_error(e)
+                      log_message("Failed to upload ongoing stats, continuing with experiment", Thor::Shell::Color::YELLOW)
                     end
-                    @exp.send_machine_stats [stats] unless stats.empty?
-                  rescue => e
-                    log_error(e)
-                    log_message("Failed to upload ongoing stats, continuing with experiment", Thor::Shell::Color::YELLOW)
                   end
                 end
               end
@@ -3406,7 +3409,7 @@ module Cnvrg
               end
 
                 # log_thread.join
-              stats_thread.join
+              stats_thread.join if docker_stats
 
                 res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average, end_time: end_time)
 
@@ -3426,7 +3429,7 @@ module Cnvrg
               log_message("Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED)
               if @exp
                 # log_thread.join
-                Thread.kill(stats_thread)
+                Thread.kill(stats_thread) if docker_stats
                 exit_status = $?.exitstatus
                 if exit_status.blank?
                   exit_status = "-1"
@@ -3450,7 +3453,7 @@ module Cnvrg
         end_commit = @project.last_local_commit
         process_running = false
         # log_thread.join
-        stats_thread.join
+        stats_thread.join if docker_stats
 
         res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
         if container
@@ -4641,6 +4644,81 @@ module Cnvrg
       end
     end
 
+    desc 'Collect and send job utilization', '', :hide => true
+    method_option :prometheus_url, :type => :string, :aliases => ["--prometheus_url"], :desc => "prometheus url to collect metrics from"
+    method_option :node_name, :type => :string, :aliases => ["--node_name"], :desc => "machie activity node name"
+    method_option :machine, :type => :boolean, :aliases => ["--machine"], :desc => "get machine_query or cluster_query"
+    method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :desc => "collect gpu metrics", :default => false
+    method_option :gaudi, :type => :boolean, :aliases => ["--gaudi"], :desc => "collect gaudi metrics", :default => false
+    method_option :wait, :type => :numeric, :aliases => ["--wait"], :desc => "to to wait between querying", :default => 30
+
+    def collect_metrics
+      @exp = Experiment.new(ENV['CNVRG_OWNER'], ENV['CNVRG_PROJECT'], job_id: ENV['CNVRG_JOB_ID'])
+      prometheus_url = options[:prometheus_url].ends_with?("/") ? options[:prometheus_url] : "#{options[:prometheus_url]}/"
+
+      translate_result = Cnvrg::API_V2.request(
+        "#{ENV['CNVRG_OWNER']}/resources/translate_metrics",
+        'GET',
+        { gpu: options[:gpu], gaudi: options[:gaudi] }
+      )
+
+      is_machine = options[:machine]
+      while true do
+        begin
+          stats = {}
+          translate_result.each do |query_name, metric|
+            if is_machine
+              metric_query = metric['machine_query'].presence || metric['query']
+              query_content = metric_query.gsub('#JOB_SLUG#', ENV['CNVRG_JOB_ID']).gsub('#NODE_NAME#', options[:node_name])
+            else
+              metric_query = metric['cluster_query'].presence || metric['query']
+              pod_name = `hostname`.strip
+              query_content = metric_query.gsub('#JOB_SLUG#', pod_name).gsub('#NODE_NAME#', options[:node_name])
+            end
+            if metric_query.blank? || query_content.blank?
+              next
+            end
+            uri = URI("#{prometheus_url}api/v1/query?query=#{query_content}")
+            resp = Net::HTTP.get(uri)
+            begin
+              result = JSON.parse(resp)
+            rescue JSON::ParserError => e
+              log_error(e)
+              next
+            end
+            data_result = result&.dig('data', 'result')
+            next unless data_result
+
+            if data_result.size > 1
+              stats[query_name] = {}
+              data_result.each_with_index do |res, i|
+                timestamp, value = res["value"]
+                uuid = res["metric"]["UUID"].presence || i
+                stat_value = value.present? ? ("%.2f" % value) : 0 # converting 34.685929244444445 to 34.69
+                stat_value = stat_value.to_i == stat_value.to_f ? stat_value.to_i : stat_value.to_f # converting 34.00 to 34
+                stats[query_name][uuid] = stat_value
+              end
+            else
+              timestamp, value = data_result&.first&.dig('value')
+              stat_value = value.present? ? ("%.2f" % value) : 0 # converting 34.685929244444445 to 34.69
+              stat_value = stat_value.to_i == stat_value.to_f ? stat_value.to_i : stat_value.to_f # converting 34.00 to 34
+              if query_name.include? 'block'
+                stats['block_io'] = {} if stats['block_io'].blank?
+                io_type = query_name.split('_')[1]
+                stats['block_io'].merge!({ io_type => stat_value })
+              else
+                stats[query_name] = stat_value
+              end
+            end
+          end
+          @exp.send_machine_stats [stats] unless stats.empty?
+        rescue => e
+          log_error(e)
+          log_message("Failed to upload ongoing stats, continuing with experiment", Thor::Shell::Color::YELLOW)
+        end
+        sleep options[:wait]
+      end
+    end
 
     desc '', '', :hide => true
 

data/lib/cnvrg/datafiles.rb

@@ -345,7 +345,8 @@ module Cnvrg
       cli = CLI.new
       cli.log_message("Using #{threads} threads with chunk size of #{chunk_size}.", Thor::Shell::Color::GREEN)
 
-      progressbar = create_progressbar("Upload Progress", files.size)
+      num_files = files.size
+      progressbar = create_progressbar("Upload Progress", num_files)
       cli = CLI.new
 
       # Vars to handle the parallelism
@@ -355,6 +356,7 @@ module Cnvrg
       dirs_queue = Queue.new
       worker_threads = []
       progress_threads = []
+      old_api = false
 
       # Vars to keep track of uploaded files and directories
       uploaded_files = []
@@ -382,23 +384,30 @@ module Cnvrg
         dir_thread = Thread.new do
           dirs_to_create = []
           loop do
-            progress_mutex.synchronize {
-              dir = dirs_queue.deq(non_block: true) rescue nil
-              dirs_to_create << dir unless dir.nil?
-            }
-            if dirs_to_create.size >= 1000 || progressbar.finished?
-              resp = Cnvrg::API.request(@base_resource + "create_dirs", "POST", {dirs: dirs_to_create, commit_sha1: commit_sha1})
-
-              break if resp == false # if resp is false it means 404 which is old server
-              unless Cnvrg::CLI.is_response_success(resp, false)
-                time = Time.current
-                Cnvrg::Logger.log_error_message("Failed to create dirs: #{time}, #{resp.try(:fetch, "message")}")
+            dir = dirs_queue.deq(non_block: true) rescue nil
+            if dir.nil? && !progressbar.finished?
+              sleep 0.2
+              Cnvrg::Logger.info("directories thread status: progressbar.finished? #{progressbar.finished?} || dirs_queue.empty? #{dirs_queue.empty?} #{dirs_queue.size}  || dirs_to_create.empty? #{dirs_to_create.empty?} #{dirs_to_create.size}")
+            else
+              dirs_to_create << dir
+
+              if dirs_to_create.size >= 1000 || progressbar.finished?
+                resp = Cnvrg::API.request(@base_resource + "create_dirs", "POST", { dirs: dirs_to_create, commit_sha1: commit_sha1 })
+                Cnvrg::Logger.info("uploaded directories chunk, finished with #{resp}")
+                if resp == false # if resp is false it means 404 which is old server
+                  old_api = true
+                  break
+                end
+                unless Cnvrg::CLI.is_response_success(resp, false)
+                  dirs_to_create = []
+                  time = Time.current
+                  Cnvrg::Logger.log_error_message("Failed to create dirs: #{time}, #{resp.try(:fetch, "message")}")
+                  next
+                end
                 dirs_to_create = []
-                next
               end
-              dirs_to_create = []
+              break if progressbar.finished? && dirs_queue.empty? && dirs_to_create.empty?
             end
-            break if progressbar.finished? && dirs_queue.empty? && dirs_to_create.empty?
           end
         end
 
@@ -409,7 +418,6 @@ module Cnvrg
               file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
               unless file.nil?
                 blob_ids = []
-
                 progress_mutex.synchronize {
                   progressbar.progress += 1
                   uploaded_files.append(file) if file[:success]
@@ -421,32 +429,31 @@ module Cnvrg
                 }
 
                 if blob_ids.present?
+                  random_id = (0...10).map { ('a'..'z').to_a[rand(26)] }.join
                   refresh_storage_token
-                  Cnvrg::Logger.info("Finished upload chunk of #{chunk_size} files, Sending Upload files save")
-
+                  Cnvrg::Logger.info("chunk #{random_id}: Finished uploading chunk of #{chunk_size} files, Sending Upload files save")
                   retry_count = 0
                   loop do
                     upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids})
 
                     if not (Cnvrg::CLI.is_response_success(upload_resp, false))
                       retry_count += 1
-                      Cnvrg::Logger.log_error_message("Failed request save files: #{Time.current}, retry: #{retry_count}")
-                      Cnvrg::Logger.info("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+                      Cnvrg::Logger.log_error_message("chunk #{random_id}: Failed request save files: #{Time.current}, retry: #{retry_count}")
                       if retry_count > 20
-                        puts "Failed to save files: #{Time.current}, trying next chunk"
+                        puts "chunk #{random_id}: Failed to save files: #{Time.current}, trying next chunk"
                         break
                       end
                       sleep 5
                       next
                     end
-                    Cnvrg::Logger.info("Chunk saved on server")
+                    Cnvrg::Logger.info("chunk #{random_id}: Chunk saved on server")
                     break
                   end
                 end
               else
                 sleep(0.1)
               end
-
+              Cnvrg::Logger.info("progress_threads: progressbar.finished? #{progressbar.finished?}")
               if progressbar.finished?
                 Cnvrg::Logger.info("Progress bar finished closing queues")
                 file_queue.close
@@ -459,35 +466,43 @@ module Cnvrg
 
         file_chunks = files.each_slice(chunk_size).to_a
         # Fetch the required files from the server:
+        num_chunks = (num_files / 1000.0).ceil
+        chunk_index = 0
         Parallel.map((file_chunks), in_threads: threads) do |chunk|
-          files_chunk = chunk.map{|p| p.gsub(/^\.\//, '')}
-          Cnvrg::Logger.info("Generating chunk idx")
+          chunk_index += 1
+          self_chunk_index = chunk_index
+          files_chunk = chunk.map { |p| p.gsub(/^\.\//, '') }
+          Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: Generating chunk idx")
           tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads, cli: cli)
+          Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: Finished Generating chunk idx")
 
-          progress_mutex.synchronize {
-            # Handle directories:
-            new_dirs = tree.keys.select { |k| tree[k].nil? }
-
-            if new_dirs.blank?
-              ## we need to send 1 file so we will inflated dirs from in case when we dont have folders in the tree
-              file = tree.keys.find { |k| tree[k] != nil }
-              dirs_queue.push file
+          # Handle directories:
+          unless old_api
+            while dirs_queue.size > 5000
+              sleep(0.1)
             end
+          end
+          new_dirs = tree.keys.select { |k| tree[k].nil? }
+          if new_dirs.blank?
+            ## we need to send 1 file so we will inflated dirs from in case when we dont have folders in the tree
+            file = tree.keys.find { |k| tree[k] != nil }
+            dirs_queue.push(file) unless old_api
+          end
+          new_dirs.each { |dir| dirs_queue.push dir }
 
-            new_dirs.each { |dir| dirs_queue.push dir }
-          }
-          Cnvrg::Logger.info("Getting files info from server")
-
+          Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: Getting files info from server")
           results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
+          Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: Finished Getting files info from server")
           next unless results
 
           if results['files'].blank?
+            Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: no files to upload skipping chunk")
             progress_mutex.synchronize { progressbar.progress += tree.keys.length }
             next
           end
 
           files_to_upload = results['files']
-
+          Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: number of files to upload in this chunk: #{tree.keys.length - files_to_upload.length}")
           progress_mutex.synchronize {
             progressbar.progress += tree.keys.length - files_to_upload.length
           }
@@ -500,10 +515,12 @@ module Cnvrg
           end
         end
 
-        Cnvrg::Logger.info("Waiting to progress and workers to finish")
+        Cnvrg::Logger.info("Waiting dir_thread to finish")
         dir_thread.join
         dirs_queue.close
+        Cnvrg::Logger.info("Waiting progress_thread to finish")
         progress_threads.each(&:join)
+        Cnvrg::Logger.info("Waiting workers to finish")
         worker_threads.each(&:join)
         Thread.report_on_exception = true
       rescue => e

data/lib/cnvrg/experiment.rb

@@ -133,23 +133,30 @@ module Cnvrg
       return response
     end
     def remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options=0,data_query=nil, image = nil, datasets = nil)
-      response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/notebook/remote", 'POST', {instance_type: instance_type,dataset_slug:data,
-                                                                                                          dataset_commit: data_commit,image_slug:image,
-                                                                                                          datasets: datasets,
-                                                                                                          commit:commit,notebook_type:notebook_type,dataset_sync_options:ds_sync_options,
-                                                                                                          dataset_query:data_query})
+      response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/notebook/remote", 'POST', { instance_type: instance_type, dataset_slug:data,
+                                                                                                           dataset_commit: data_commit, image_slug:image,
+                                                                                                           datasets: datasets,
+                                                                                                           commit:commit, notebook_type:notebook_type, dataset_sync_options:ds_sync_options,
+                                                                                                           dataset_query: data_query })
       return response
     end
 
     def upload_temp_log(temp_log)
-      response = Cnvrg::API.request(@base_resource + "experiment/upload_temp_log", 'POST', {output: temp_log,
-                                                                                            exp_slug: @slug})
-      Cnvrg::CLI.is_response_success(response,false)
+      response = Cnvrg::API.request(@base_resource + "experiment/upload_temp_log", 'POST', { output: temp_log,
+                                                                                             exp_slug: @slug })
+      Cnvrg::CLI.is_response_success(response, false)
     end
 
     def send_machine_stats(stats)
-      response = Cnvrg::API.request(@base_resource + "experiment/upload_stats", "POST", {exp_slug: @slug, stats: stats.map{|s| s.merge!({time: Time.now})}})
-      Cnvrg::CLI.is_response_success(response,false)
+      response = Cnvrg::API.request(
+        @base_resource + "experiment/upload_stats",
+        "POST",
+        {
+          exp_slug: @slug,
+          stats: stats.map { |s| s.merge!({ time: Time.now }) }
+        }
+      )
+      Cnvrg::CLI.is_response_success(response, false)
     end
 
     def end(output, exit_status, end_commit, cpu_average, memory_average, end_time: nil)
@@ -158,7 +165,7 @@ module Cnvrg
       success = false
       end_time ||= Time.now
       while tries < 10 and success.blank?
-        sleep (tries*rand) ** 2 ### exponential backoff
+        sleep (tries * rand) ** 2 ### exponential backoff
                                 ## this call is super important so we cant let it crash.
 
         tries += 1

data/lib/cnvrg/files.rb

@@ -134,7 +134,6 @@ module Cnvrg
           end
         end
 
-
       blob_ids.concat blob_id_chunk
     end
 

data/lib/cnvrg/helpers.rb

@@ -14,6 +14,7 @@ module Cnvrg
       }
     end
     def checkmark
+      return "" if Cnvrg::Helpers.windows?
       checkmark = "\u2713"
       return checkmark.encode('utf-8')
     end

data/lib/cnvrg/helpers/executer.rb

@@ -201,7 +201,7 @@ class Cnvrg::Helpers::Executer
     pod_name = `hostname`.strip rescue nil
     node_name = nil
     if pod_name.present?
-      pod_describe = `kubectl -n cnvrg get pod #{pod_name} -o json` rescue nil
+      pod_describe = `kubectl get pod #{pod_name} -o json` rescue nil
       pod_describe = JSON.parse(pod_describe) rescue {}
       node_name = pod_describe["spec"]["nodeName"] rescue nil
     end
@@ -217,7 +217,7 @@ class Cnvrg::Helpers::Executer
 
   def get_pod_events(pod_name)
     return if pod_name.blank?
-    `kubectl get event --namespace cnvrg --field-selector involvedObject.name=#{pod_name} -o json`
+    `kubectl get event --field-selector involvedObject.name=#{pod_name} -o json`
   end
 
   def get_node_events(node_name)

data/lib/cnvrg/project.rb

@@ -381,7 +381,7 @@ module Cnvrg
     def generate_output_dir(output_dir)
       Cnvrg::Logger.log_info("Generating output dir for #{output_dir}")
       upload_list = []
-      list = Dir.glob("#{output_dir}/**/*", File::FNM_DOTMATCH)
+      list = Dir.glob("/cnvrg/#{output_dir}/**/*", File::FNM_DOTMATCH)
       Parallel.map(list, in_threads: IDXParallelThreads) do |e|
         next if e.end_with? "/."
         if File.directory? e
@@ -517,17 +517,17 @@ module Cnvrg
       commit = local_idx[:commit]
       tree = local_idx[:tree]
       ignore_list = self.send_ignore_list()
-      if force
+      if force or specific_files.present?
         added = []
         if tree.present?
           added += local_idx[:tree].keys
         end
-        response = {"result" => {"commit" => nil, "tree" => {"added" => added,
-                                                             "updated_on_server" => [],
-                                                             "updated_on_local" => [],
-                                                             "update_local" => [],
-                                                             "deleted" => [],
-                                                             "conflicts" => []}}}
+        response = { "result" => { "commit" => nil, "tree" => { "added" => added,
+                                                                "updated_on_server" => [],
+                                                                "updated_on_local" => [],
+                                                                "update_local" => [],
+                                                                "deleted" => [],
+                                                                "conflicts" => [] } } }
         return response
       end
       #we dont want to send it on download - we only compare between commits sha1 in download.
@@ -535,6 +535,7 @@ module Cnvrg
         #the new server doesnt need the tree, but the old probably needs :X
         local_idx[:tree] = {} if Cnvrg::Helpers.server_version > 0
       end
+
       response = Cnvrg::API.request(@base_resource + "status", 'POST', {idx: local_idx, new_branch: new_branch,
                                                                         current_commit: commit, ignore: ignore_list, force: force, in_exp: in_exp, download: download})
 

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.11.25'
+  VERSION = '1.11.31'
 end

metadata

@@ -1,16 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.11.25
+  version: 1.11.31
 platform: ruby
 authors:
 - Yochay Ettun
 - Leah Kolben
-- Omer Shacham
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-27 00:00:00.000000000 Z
+date: 2021-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -96,6 +95,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.9.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.9.10
 - !ruby/object:Gem::Dependency
   name: mimemagic
   requirement: !ruby/object:Gem::Requirement
@@ -105,7 +124,7 @@ dependencies:
         version: 0.3.1
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.2
+        version: 0.3.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -115,7 +134,7 @@ dependencies:
         version: 0.3.1
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.2
+        version: 0.3.7
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -394,6 +413,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- Readme.md
 - bin/cnvrg
 - cnvrg.gemspec
 - lib/cnvrg.rb