cnvrg 2.1.12 → 2.1.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 850c099310fb739283b4fe34c0c0eccfb9bd4297e9d525daa497eb45481ea707
-  data.tar.gz: d84a2854fe5ede6ab5a65804d11068a00c88dccf5d341a9d54728c3f5870a9b0
+  metadata.gz: 944b6505221d4431fb485095b6005a477376eedbe8f3eeb719e8b91fbce95c1b
+  data.tar.gz: 22b865c7df6e46610041c9e5c2e0d496408e422588955a141cea14926e30c2cb
 SHA512:
-  metadata.gz: 0e3da1a344367dacf8496c0e59de07770ce69db0ab59009d14fd27bc15ae6e9e56bb653e49c95d732a5b1dde962a550cba8da3310806c615015fa693ae9899d3
-  data.tar.gz: 01eda251528f076aa244558272ef4a630145fb7e8752f5f25b7c1f6bfb39be61e33189ad1e8b5e50ac1d80825de303ec479186eb65b0db6d2b62cce1b693c99a
+  metadata.gz: 4396183d600780db95f3cffad3495593d7cd388a37779b160f199cd33ddeb85f4a4deb20133e1ea6932663d80088da3f14e2f4f590938c2b80d8c921044b69aa
+  data.tar.gz: 739e381136764fc68b95cf45007c1281b509b814626a769bf7ffe76369d99f94b08880b3b3f1b71fec011a0b299e36d9b6906d8a520eee4b1fd18b0eeca57f64

data/Readme.md

@@ -117,4 +117,37 @@
 * DEV-16090 - Bug: Tensorboard compare - webapp gets stuck in "init" state 
 ## Version v2.1.12
 2022-11-09
-* DEV-15972 - Bug: Customer cannot use dataset query due to ssl error
+* DEV-15972 - Bug: Customer cannot use dataset query due to ssl error
+## Version v2.1.13
+2023-03-05
+* DEV-16372 - Epic: Cli V2 release - phase 1
+h2. 🎯 Objective
+
+Release v2 of cnvrg cli and deprecate the old version
+
+h2. 🤔 Assumptions
+
+# *Docs will be released (joint effort of tech writer, support and dev)*
+# *cnvrg:v6 image will be released (with updated python version)*
+# *All issues on High & Highest priority n sdk-productionize fixVersion will be resolved*
+# *QA tests will be performed in different set ups (aks/eks/gke, windows device, mac m1)*
+
+
+Note: this is not the most detailed epic, as most things already exist in cliv1 or sdkv2. for any questions please contact [~accountid:5fb5461f4a09640069dbf768] 
+* DEV-16372 - Epic: Cli V2 release - phase 1
+h2. 🎯 Objective
+
+Release v2 of cnvrg cli and deprecate the old version
+
+h2. 🤔 Assumptions
+
+# *Docs will be released (joint effort of tech writer, support and dev)*
+# *cnvrg:v6 image will be released (with updated python version)*
+# *All issues on High & Highest priority n sdk-productionize fixVersion will be resolved*
+# *QA tests will be performed in different set ups (aks/eks/gke, windows device, mac m1)*
+
+
+Note: this is not the most detailed epic, as most things already exist in cliv1 or sdkv2. for any questions please contact [~accountid:5fb5461f4a09640069dbf768] 
+## Version v2.1.14
+2023-05-29
+* DEV-18350 - Bug: Error occured, undefined method `[]' for false:FalseClass is displayed when running any command inside the debug mode of experiment

data/cnvrg.gemspec

@@ -47,4 +47,5 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'activesupport', '~> 5.2.0'
     spec.add_runtime_dependency 'ruby-progressbar'
     spec.add_runtime_dependency 'down'
+    spec.add_runtime_dependency 'nokogiri', '~> 1.13.10'
 end

data/lib/cnvrg/cli.rb

@@ -3636,19 +3636,15 @@ module Cnvrg
           end
 
           log_message(str, Thor::Shell::Color::GREEN)
-          
+
           exit_status = 0
-          
+
           if options['wait']
             end_pos = 0
             while true
               tries = 0
               begin
-                result = Cnvrg::API_V2.request(
-                    "#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}/info",
-                    'GET',
-                    { exit_status: true, grid: res["result"]["grid"], pos: end_pos }
-                )
+                result = fetch_experiment_info(res,project,end_pos)
 
                 exit_statuses = result.values.pluck('exit_status')
                 if exit_statuses.include? nil
@@ -5956,6 +5952,24 @@ module Cnvrg
         end
       end
 
+
+      def fetch_experiment_info(res, project, end_pos)
+
+        result =
+          Cnvrg::API_V2
+            .request("#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}/info",
+          'GET',
+          { exit_status: true, grid: res["result"]["grid"], pos: end_pos }
+        )
+
+        if result.key?("exit_status")
+          return result
+        end
+
+        return result["data"]["attributes"]["info"]
+
+      end
+
     end
   end
 

data/lib/cnvrg/dataset.rb

@@ -13,7 +13,7 @@ module Cnvrg
         if project_home.present?
           @local_path = project_home
           @working_dir = project_home
-          config = YAML.load_file(project_home + "/.cnvrg/config.yml")
+          config = Cnvrg::Helpers.get_config_v2_dataset(project_home)
           @title = config[:dataset_name]
           @slug = config[:dataset_slug]
           @owner = config[:owner]

data/lib/cnvrg/helpers.rb

@@ -2,17 +2,19 @@ module Cnvrg
   module Helpers
 
     extend self
+
     def parallel_threads
       15
     end
 
     def self.parallel_options
       {
-          in_processes: Cnvrg::CLI::ParallelProcesses,
-          in_thread: Cnvrg::CLI::ParallelThreads,
-          isolation: true
+        in_processes: Cnvrg::CLI::ParallelProcesses,
+        in_thread: Cnvrg::CLI::ParallelThreads,
+        isolation: true
       }
     end
+
     def checkmark
       return "" if Cnvrg::Helpers.windows?
       checkmark = "\u2713"
@@ -47,8 +49,8 @@ module Cnvrg
       home_dir = File.expand_path('~')
       config = {}
       begin
-        if File.exist? home_dir+"/.cnvrg/config.yml"
-          config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+        if File.exist? home_dir + "/.cnvrg/config.yml"
+          config = YAML.load_file(home_dir + "/.cnvrg/config.yml")
         end
       end
       return config
@@ -56,17 +58,16 @@ module Cnvrg
 
     def set_config(config)
       home_dir = File.expand_path('~')
-      File.open("#{home_dir}/.cnvrg/config.yml", "w"){|f| f.write config.to_yaml }
+      File.open("#{home_dir}/.cnvrg/config.yml", "w") { |f| f.write config.to_yaml }
       return config
     end
 
-
     def remote_url
       home_dir = File.expand_path('~')
       config = ""
       begin
-        if File.exist? home_dir+"/.cnvrg/config.yml"
-          config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+        if File.exist? home_dir + "/.cnvrg/config.yml"
+          config = YAML.load_file(home_dir + "/.cnvrg/config.yml")
         else
           return "https://app.cnvrg.io"
         end
@@ -99,10 +100,10 @@ module Cnvrg
       home_dir = File.expand_path('~')
       config = ""
       begin
-        if File.exist? home_dir+"/.cnvrg/config.yml"
-          config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+        if File.exist? home_dir + "/.cnvrg/config.yml"
+          config = YAML.load_file(home_dir + "/.cnvrg/config.yml")
         else
-         return true
+          return true
 
         end
 
@@ -262,105 +263,105 @@ parameters:
       Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond)
     end
 
-    def decrypt(key,iv,str)
+    def decrypt(key, iv, str)
       begin
 
-      cipher = OpenSSL::Cipher.new("aes-256-cbc").decrypt
-      cipher.key = key
-      cipher.iv = Base64.decode64 iv.encode('utf-8')
+        cipher = OpenSSL::Cipher.new("aes-256-cbc").decrypt
+        cipher.key = key
+        cipher.iv = Base64.decode64 iv.encode('utf-8')
 
-      result = Base64.decode64 (str.encode('utf-8'))
-      result = cipher.update(result)
-      result << cipher.final
-      return result.force_encoding('utf-8')
+        result = Base64.decode64 (str.encode('utf-8'))
+        result = cipher.update(result)
+        result << cipher.final
+        return result.force_encoding('utf-8')
 
-      # return result
+        # return result
       rescue => e
         puts e
 
-
       end
 
-
     end
 
     # memory
     #
-    def get_mem(pid)
-    end
+    def get_mem(pid) end
 
-    def get_s3_props(files) #will return client and decryptor
+    def get_s3_props(files)
+      #will return client and decryptor
       sts_path = files["path_sts"]
       retries = 0
-      success= false
+      success = false
       while !success and retries < 20
         begin
           if !Helpers.is_verify_ssl
-            body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
+            body = open(sts_path, { ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE }).read
           else
             body = open(sts_path).read
           end
           success = true
         rescue => e
-          retries +=1
+          retries += 1
           sleep(5)
 
         end
       end
       if !success
-        return Cnvrg::Result.new(false,"couldn't download some files", "error in sts", "" )
+        return Cnvrg::Result.new(false, "couldn't download some files", "error in sts", "")
       end
       split = body.split("\n")
       key = split[0]
       iv = split[1]
 
-      access =  Cnvrg::Helpers.decrypt(key, iv, files["sts_a"])
+      access = Cnvrg::Helpers.decrypt(key, iv, files["sts_a"])
 
-      secret =  Cnvrg::Helpers.decrypt(key,iv, files["sts_s"])
+      secret = Cnvrg::Helpers.decrypt(key, iv, files["sts_s"])
 
-      session =  Cnvrg::Helpers.decrypt(key,iv, files["sts_st"])
-      region =  Cnvrg::Helpers.decrypt(key,iv, files["region"])
+      session = Cnvrg::Helpers.decrypt(key, iv, files["sts_st"])
+      region = Cnvrg::Helpers.decrypt(key, iv, files["region"])
 
-      bucket =  Cnvrg::Helpers.decrypt(key,iv, files["bucket"])
+      bucket = Cnvrg::Helpers.decrypt(key, iv, files["bucket"])
       is_s3 = files["is_s3"]
-      server_side_encryption =files["server_side_encryption"]
+      server_side_encryption = files["server_side_encryption"]
 
       if is_s3 or is_s3.nil?
         client = Aws::S3::Client.new(
-            :access_key_id =>access,
-            :secret_access_key => secret,
-            :session_token => session,
-            :region => region,
-            :http_open_timeout => 60, :retry_limit => 20)
+          :access_key_id => access,
+          :secret_access_key => secret,
+          :session_token => session,
+          :region => region,
+          :http_open_timeout => 60, :retry_limit => 20)
         use_accelerate_endpoint = true
       else
 
-        endpoint = Cnvrg::Helpers.decrypt(key,iv, files["endpoint"])
+        endpoint = Cnvrg::Helpers.decrypt(key, iv, files["endpoint"])
         client = Aws::S3::Client.new(
-            :access_key_id =>access,
-            :secret_access_key => secret,
-            :region => region,
-            :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
-            :http_open_timeout => 60, :retry_limit => 20)
+          :access_key_id => access,
+          :secret_access_key => secret,
+          :region => region,
+          :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
+          :http_open_timeout => 60, :retry_limit => 20)
         use_accelerate_endpoint = false
       end
 
       if !server_side_encryption
-        upload_options = {:use_accelerate_endpoint => use_accelerate_endpoint}
+        upload_options = { :use_accelerate_endpoint => use_accelerate_endpoint }
       else
-        upload_options = {:use_accelerate_endpoint => use_accelerate_endpoint, :server_side_encryption => server_side_encryption}
+        upload_options = { :use_accelerate_endpoint => use_accelerate_endpoint, :server_side_encryption => server_side_encryption }
       end
-      return {client: client, key: key, iv: iv, bucket: bucket, upload_options: upload_options}
+      return { client: client, key: key, iv: iv, bucket: bucket, upload_options: upload_options }
     end
 
     def get_experiment_events_log_from_server(exp, project, commit: nil)
       dest_dir = exp["slug"]
       commit = commit || exp["end_commit"]
       response = project.clone(0, commit)
-      Cnvrg::CLI.is_response_success(response, should_exit=false)
+      Cnvrg::CLI.is_response_success(response, should_exit = false)
       commit_sha1 = response["result"]["commit"]
       files = response["result"]["tree"].keys
-      files = files.select do |f| f.include?("tfevents") end
+      files = files.select do |f|
+        f.include?("tfevents")
+      end
       @files = Cnvrg::Files.new(project.owner, project.slug, project_home: "", project: project)
       @files.download_files(files, commit_sha1, progress: nil)
       FileUtils.rm_rf("#{dest_dir}")
@@ -374,6 +375,33 @@ parameters:
       end
       return num_of_new_files
     end
-  end
 
+    def get_config_v2_project(dir, owner, slug)
+      config = {}
+      if File.exist? dir + "/.cnvrg/config.yml"
+        config = YAML.load_file(dir + "/.cnvrg/config.yml")
+      elsif File.exist? dir + "/.cnvrg/cnvrg.config"
+        cnvrgv2_config = YAML.load_file(dir + "/.cnvrg/cnvrg.config")
+        config[:project_name] = cnvrgv2_config["project_slug"]
+        config[:project_slug] = cnvrgv2_config["project_slug"]
+        config[:owner] = ENV['CNVRG_OWNER']
+        config[:git] = cnvrgv2_config["git"] || false
+      else
+        return { owner: owner, project_slug: slug }
+      end
+      config
+    end
+    def get_config_v2_dataset(dir)
+      config = {}
+      if File.exist? dir + "/.cnvrg/config.yml"
+        config = YAML.load_file(dir + "/.cnvrg/config.yml")
+      elsif File.exist? dir + "/.cnvrg/cnvrg.config"
+        cnvrgv2_config = YAML.load_file(dir + "/.cnvrg/cnvrg.config")
+        config[:dataset_name] = cnvrgv2_config["dataset_slug"]
+        config[:dataset_slug] = cnvrgv2_config["dataset_slug"]
+        config[:owner] = ENV['CNVRG_OWNER']
+      end
+      config
+    end
+  end
 end

data/lib/cnvrg/project.rb

@@ -8,12 +8,12 @@ module Cnvrg
     RemoteURL ||= "https://cnvrg.io"
     IDXParallelThreads ||= Cnvrg::Helpers.parallel_threads
 
-
-    def initialize(project_home=nil, slug: nil, owner: nil)
+    def initialize(project_home = nil, slug: nil, owner: nil)
       begin
         @local_path = project_home
         @working_dir = project_home
-        config = YAML.load_file(project_home + "/.cnvrg/config.yml") rescue {owner: owner, project_slug: slug}
+        # read from env and new config file
+        config = Cnvrg::Helpers.get_config_v2_project(project_home, owner, slug)
         @title = config[:project_name]
         @slug = config[:project_slug]
         @owner = config[:owner]
@@ -29,12 +29,10 @@ module Cnvrg
       "users/#{@owner}/projects/#{@slug}/"
     end
 
-
     def last_local_commit
       YAML.load_file(@local_path + "/.cnvrg/idx.yml")[:commit] rescue nil
     end
 
-
     def url
       url = Cnvrg::Helpers.remote_url
       "#{url}/#{self.owner}/projects/#{self.slug}"
@@ -142,11 +140,10 @@ module Cnvrg
         ]
       end
 
-
       list_files = [
-          project_name + "/README.md",
-          project_name + "/.cnvrgignore",
-          project_name + "/.cnvrg/config.yml"
+        project_name + "/README.md",
+        project_name + "/.cnvrgignore",
+        project_name + "/.cnvrg/config.yml"
       ]
       cnvrgreadme = Helpers.readme_content
       cnvrgignore = Helpers.cnvrgignore_content
@@ -155,22 +152,22 @@ module Cnvrg
       begin
 
         owner = Cnvrg::CLI.get_owner()
-        response = Cnvrg::API.request("cli/create_project", 'POST', {title: project_name, owner: owner, is_docker: with_docker, bucket: bucket})
+        response = Cnvrg::API.request("cli/create_project", 'POST', { title: project_name, owner: owner, is_docker: with_docker, bucket: bucket })
         Cnvrg::CLI.is_response_success(response)
         response = JSON.parse response["result"]
         project_slug = response["slug"]
 
-        config = {project_name: project_name,
-                  project_slug: project_slug,
-                  owner: owner,
-                  docker: with_docker}
+        config = { project_name: project_name,
+                   project_slug: project_slug,
+                   owner: owner,
+                   docker: with_docker }
         FileUtils.mkdir_p list_dirs
         FileUtils.touch list_files
 
-        File.open(project_name + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
-        File.open(project_name + "/.cnvrgignore", "w+") {|f| f.write cnvrgignore}
-        File.open(project_name + "/README.md", "w+") {|f| f.write cnvrgreadme}
-        File.open(project_name + "/src/hyper.yaml", "w+") {|f| f.write cnvrghyper}
+        File.open(project_name + "/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+        File.open(project_name + "/.cnvrgignore", "w+") { |f| f.write cnvrgignore }
+        File.open(project_name + "/README.md", "w+") { |f| f.write cnvrgreadme }
+        File.open(project_name + "/src/hyper.yaml", "w+") { |f| f.write cnvrghyper }
 
       rescue
         return false
@@ -183,32 +180,32 @@ module Cnvrg
       list_dirs = [".cnvrg"
       ]
       list_files = [
-          ".cnvrg/config.yml"
+        ".cnvrg/config.yml"
       ]
       if !ignore_exits
         list_files <<
-            ".cnvrgignore"
+          ".cnvrgignore"
       end
 
       cnvrgreadme = Helpers.readme_content
       cnvrgignore = Helpers.cnvrgignore_content
       begin
-        response = Cnvrg::API.request("cli/create_project", 'POST', {title: project_name, owner: owner, is_docker: docker, bucket: bucket})
+        response = Cnvrg::API.request("cli/create_project", 'POST', { title: project_name, owner: owner, is_docker: docker, bucket: bucket })
         Cnvrg::CLI.is_response_success(response)
         response = JSON.parse response["result"]
         project_slug = response["slug"]
 
-        config = {project_name: project_name,
-                  project_slug: project_slug,
-                  owner: owner,
-                  git: git}
+        config = { project_name: project_name,
+                   project_slug: project_slug,
+                   owner: owner,
+                   git: git }
         FileUtils.mkdir_p list_dirs
         FileUtils.touch list_files
-        File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
-        File.open(".cnvrgignore", "w+") {|f| f.write cnvrgignore} unless ignore_exits
+        File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+        File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless ignore_exits
         if !File.exist? "README" and !File.exist? "README.md"
           FileUtils.touch ["README.md"]
-          File.open("README.md", "w+") {|f| f.write cnvrgreadme}
+          File.open("README.md", "w+") { |f| f.write cnvrgreadme }
         end
 
       rescue => e
@@ -223,23 +220,21 @@ module Cnvrg
                    project_name + "/.cnvrg"
       ]
 
-
       list_files = [
-          project_name + "/.cnvrg/config.yml",
-          project_name + "/.cnvrgignore",
+        project_name + "/.cnvrg/config.yml",
+        project_name + "/.cnvrgignore",
       ]
       begin
-        config = {project_name: project_name,
-                  project_slug: project_slug,
-                  owner: project_owner,
-                  git: is_git}
+        config = { project_name: project_name,
+                   project_slug: project_slug,
+                   owner: project_owner,
+                   git: is_git }
         FileUtils.mkdir_p list_dirs
         FileUtils.touch list_files
         cnvrgignore = Helpers.cnvrgignore_content
 
-
-        File.open(project_name + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
-        File.open(project_name + "/.cnvrgignore", "w+") {|f| f.write cnvrgignore}
+        File.open(project_name + "/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+        File.open(project_name + "/.cnvrgignore", "w+") { |f| f.write cnvrgignore }
 
       rescue
         return false
@@ -256,11 +251,11 @@ module Cnvrg
       if !ignore_exits
         begin
           list_files = [
-              path
+            path
           ]
           FileUtils.touch list_files
           cnvrgignore = Helpers.cnvrgignore_content
-          File.open(path, "w+") {|f| f.write cnvrgignore}
+          File.open(path, "w+") { |f| f.write cnvrgignore }
         rescue => e
           return false
         end
@@ -272,32 +267,28 @@ module Cnvrg
       cli = Cnvrg::CLI.new()
       begin
         list_dirs = [
-            ".cnvrg"
+          ".cnvrg"
         ]
 
-
         list_files = [
-            ".cnvrg/config.yml",
+          ".cnvrg/config.yml",
 
         ]
 
-
-        config = {project_name: project_name,
-                  project_slug: project_slug,
-                  owner: project_owner,
-                  git: is_git
+        config = { project_name: project_name,
+                   project_slug: project_slug,
+                   owner: project_owner,
+                   git: is_git
         }
         FileUtils.mkdir_p list_dirs
         FileUtils.touch list_files
 
-
-        File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
+        File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
         if !File.exist? ".cnvrgignore"
           FileUtils.touch ".cnvrgignore"
           list_files << ".cnvrgignore"
           cnvrgignore = Helpers.cnvrgignore_content
-          File.open(".cnvrgignore", "w+") {|f| f.write cnvrgignore}
-
+          File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore }
 
         end
         true
@@ -311,7 +302,7 @@ module Cnvrg
     def update_is_new_branch(new_branch)
       config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
       config[:new_branch] = new_branch
-      File.open(@working_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
+      File.open(@working_dir + "/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
     end
 
     def get_new_branch
@@ -347,7 +338,7 @@ module Cnvrg
     end
 
     def get_storage_client_fallback
-      response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/download_files", "POST", {files: [], commit: get_latest_commit})
+      response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/download_files", "POST", { files: [], commit: get_latest_commit })
       raise StandardError.new("Can't find project credentials") unless Cnvrg::CLI.is_response_success(response, false)
       files = response['result']
       storage = files['is_s3'] ? 's3' : 'minio'
@@ -368,13 +359,13 @@ module Cnvrg
       if slug.blank? or owner.blank? or name.blank?
         return
       end
-      File.open(@working_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
+      File.open(@working_dir + "/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
     end
 
     def remove_new_branch
       config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
       new_config = config.except(:new_branch)
-      File.open(@working_dir + "/.cnvrg/config.yml", "w+") {|f| f.write new_config.to_yaml}
+      File.open(@working_dir + "/.cnvrg/config.yml", "w+") { |f| f.write new_config.to_yaml }
     end
 
     def generate_git_diff
@@ -424,16 +415,15 @@ module Cnvrg
       #   upload_list << output_dir + "/"
       # end
 
-
       return upload_list
 
     end
 
     def generate_idx(deploy: false, files: [])
       if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
-        old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml") rescue {:tree => {}, :commit => nil}
+        old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml") rescue { :tree => {}, :commit => nil }
       else
-        old_idx = {:tree => {}, :commit => nil}
+        old_idx = { :tree => {}, :commit => nil }
       end
 
       tree_idx = Hash.new(0)
@@ -441,7 +431,7 @@ module Cnvrg
       ### if file specified, just take them, dont calculate everything from scratch
       list_ignore = self.get_ignore_list()
       if files.blank?
-        list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject {|x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~ /^#{self.local_path}\/\.git\/*/) or (x =~ /^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~ /^#{self.local_path}\/\.cnvrgignore/)}
+        list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~ /^#{self.local_path}\/\.git\/*/) or (x =~ /^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~ /^#{self.local_path}\/\.cnvrgignore/) }
       else
         list = files
       end
@@ -449,7 +439,7 @@ module Cnvrg
         list_ignore += ["main.py", "main.pyc", "__init__.py", "uwsgi.ini"]
         list_ignore.flatten!
       end
-      list_ignore_new = list_ignore.map{|x| x.gsub("//","/")} rescue []
+      list_ignore_new = list_ignore.map { |x| x.gsub("//", "/") } rescue []
       # list.each do |e|
       Parallel.map(list, in_threads: IDXParallelThreads) do |e|
         label = e.sub(self.local_path + "/", "")
@@ -474,23 +464,23 @@ module Cnvrg
           end
 
           if old_idx.nil? or old_idx.to_h[:tree].nil?
-            tree_idx[label] = {sha1: sha1, commit_time: nil, last_modified: last_modified}
+            tree_idx[label] = { sha1: sha1, commit_time: nil, last_modified: last_modified }
           elsif file_in_idx.nil? or file_in_idx[:sha1] != sha1 or file_in_idx[:last_modified].blank? or file_in_idx[:last_modified] != last_modified
-            tree_idx[label] = {sha1: sha1, commit_time: nil, last_modified: last_modified}
+            tree_idx[label] = { sha1: sha1, commit_time: nil, last_modified: last_modified }
           else
             tree_idx[label] = old_idx[:tree][label]
           end
         end
       end
       old_idx[:tree] = tree_idx
-      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write old_idx.to_yaml}
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write old_idx.to_yaml }
       return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
     end
 
     def get_idx
       unless File.exists? "#{self.local_path}/.cnvrg/idx.yml"
-        empty_idx = {:commit => nil, :tree => {}}
-        File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write empty_idx.to_yaml}
+        empty_idx = { :commit => nil, :tree => {} }
+        File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write empty_idx.to_yaml }
         return empty_idx
       end
       YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
@@ -498,16 +488,16 @@ module Cnvrg
 
     def set_idx(idx)
       FileUtils.mkdir_p("#{self.local_path}/.cnvrg")
-      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx.to_yaml}
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
     end
 
     def clone(remote = 0, commit)
-      response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/clone", 'POST', {project_slug: self.slug, remote: remote, commit: commit})
+      response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/clone", 'POST', { project_slug: self.slug, remote: remote, commit: commit })
       return response
     end
 
     def git_download_commit(commit)
-      response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/git_download_commit", 'POST', {commit_sha1: commit})
+      response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/git_download_commit", 'POST', { commit_sha1: commit })
       CLI.is_response_success(response, true)
       return response
     end
@@ -519,7 +509,7 @@ module Cnvrg
       return commit
     end
 
-    def compare_idx(new_branch, force:false, deploy: false, in_exp:false, specific_files: [], download: false)
+    def compare_idx(new_branch, force: false, deploy: false, in_exp: false, specific_files: [], download: false)
       is_download = download
       if is_download
         local_idx = self.get_idx
@@ -550,8 +540,8 @@ module Cnvrg
         local_idx[:tree] = {} if Cnvrg::Helpers.server_version > 0
       end
 
-      response = Cnvrg::API.request(@base_resource + "status", 'POST', {idx: local_idx, new_branch: new_branch,
-                                                                        current_commit: commit, ignore: ignore_list, force: force, in_exp: in_exp, download: download})
+      response = Cnvrg::API.request(@base_resource + "status", 'POST', { idx: local_idx, new_branch: new_branch,
+                                                                         current_commit: commit, ignore: ignore_list, force: force, in_exp: in_exp, download: download })
 
       CLI.is_response_success(response, true)
       if is_download
@@ -565,13 +555,13 @@ module Cnvrg
         end
         @files = self.get_files
         local_tree = @files.calculate_sha1(resolve.keys)
-        changed_files = resolve.keys.select {|file| local_tree[file] != resolve[file]}
+        changed_files = resolve.keys.select { |file| local_tree[file] != resolve[file] }
 
         # means that the user changed the file locally
         response['result']['tree']['update_local'] = changed_files
 
         # means that we already downloaded this file and we dont need it anymore
-        downloaded_files = destination_files.keys.select {|file| local_tree[file] == destination_files[file]}
+        downloaded_files = destination_files.keys.select { |file| local_tree[file] == destination_files[file] }
         response['result']['tree']['added'] -= downloaded_files
         response['result']['tree']['updated_on_server'] -= downloaded_files
       end
@@ -590,8 +580,8 @@ module Cnvrg
       tree = local_idx[:tree]
       response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/jump_to",
                                     'POST',
-                                    {tree: tree, ignore: ignore_list,
-                                     dest_commit: destination, current_commit: current_commit})
+                                    { tree: tree, ignore: ignore_list,
+                                      dest_commit: destination, current_commit: current_commit })
       CLI.is_response_success(response, false)
       response
     end
@@ -601,7 +591,7 @@ module Cnvrg
       @is_branch = !is_latest
       config[:is_branch] = @is_branch
 
-      File.open(@working_dir + "/.cnvrg/config.yml", 'w') {|f| f.write config.to_yaml}
+      File.open(@working_dir + "/.cnvrg/config.yml", 'w') { |f| f.write config.to_yaml }
 
     end
 
@@ -609,7 +599,7 @@ module Cnvrg
       if commit.nil? or commit.empty?
         commit = last_local_commit
       end
-      response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
+      response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', { current_commit: commit })
       CLI.is_response_success(response, false)
       update_is_new_branch(response["result"]["new_branch"])
       return response["result"]["new_branch"]
@@ -621,20 +611,20 @@ module Cnvrg
       files.each do |path|
         idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
       end
-      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
 
       return true
     end
 
     def deploy(file_to_run, function, input_params, commit_to_run, instance_type, image_slug, scheduling_query, local_timestamp, workers, file_input, title)
-      response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/deploy", 'POST', {file_to_run: file_to_run, function: function,
-                                                                                         image_slug: image_slug, input_params: input_params,
-                                                                                         commit_sha1: commit_to_run,
-                                                                                         instance_type: instance_type,
-                                                                                         scheduling_query: scheduling_query,
-                                                                                         local_timestamp: local_timestamp,
-                                                                                         workers: workers, file_input: file_input,
-                                                                                         title: title})
+      response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/deploy", 'POST', { file_to_run: file_to_run, function: function,
+                                                                                          image_slug: image_slug, input_params: input_params,
+                                                                                          commit_sha1: commit_to_run,
+                                                                                          instance_type: instance_type,
+                                                                                          scheduling_query: scheduling_query,
+                                                                                          local_timestamp: local_timestamp,
+                                                                                          workers: workers, file_input: file_input,
+                                                                                          title: title })
       return response
     end
 
@@ -644,23 +634,21 @@ module Cnvrg
       return response
     end
 
-
     def get_experiments
       response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/experiments/list", 'GET')
       CLI.is_response_success(response)
       return response
     end
 
-
     def get_experiment(slug)
       response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/experiments/#{slug}", 'GET')
-      response['status']=200
+      response['status'] = 200
       CLI.is_response_success(response)
       return response
     end
 
     def fetch_webapp_slugs(webapp_slug, slugs: nil)
-      response = Cnvrg::API_V2.request("#{self.owner}/projects/#{self.slug}/webapps/#{webapp_slug}" , 'GET')
+      response = Cnvrg::API_V2.request("#{self.owner}/projects/#{self.slug}/webapps/#{webapp_slug}", 'GET')
 
       if response.key?("experiments")
         return response["experiments"]
@@ -675,7 +663,7 @@ module Cnvrg
       idx_hash[:commit] = commit
       self.set_on_branch(latest) unless latest.nil?
 
-      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
       return true
     end
 
@@ -696,12 +684,11 @@ module Cnvrg
       @machines || []
     end
 
-
     def update_job_jupyter_token(job_type, job_id, token)
       owner = self.owner || ENV['CNVRG_OWNER']
       slug = self.slug || ENV['CNVRG_PROJECT']
       base_url = "users/#{owner}/projects/#{slug}/jobs/#{job_type.underscore}/#{job_id}"
-      Cnvrg::API.request("#{base_url}/update_jupyter_token", "POST", {token: token})
+      Cnvrg::API.request("#{base_url}/update_jupyter_token", "POST", { token: token })
     end
 
     def check_machine(machine)
@@ -711,7 +698,6 @@ module Cnvrg
       machines.include? machine
     end
 
-
     def fetch_project
       resp = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/get_project", "GET")
       res = JSON.parse(resp['result']) rescue nil
@@ -733,12 +719,11 @@ module Cnvrg
         raise StandardError.new("Cant find job env variables")
       end
       logs.each_slice(10).each do |temp_logs|
-        Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/log", "POST", {job_type: job_type, job_id: job_id, logs: temp_logs, log_level: level, step: step, timestamp: Time.now})
+        Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/log", "POST", { job_type: job_type, job_id: job_id, logs: temp_logs, log_level: level, step: step, timestamp: Time.now })
         sleep(1)
       end
     end
 
-
     def job_commands
       job_type, job_id = ENV['CNVRG_JOB_TYPE'], ENV['CNVRG_JOB_ID']
       resp = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/commands", "GET")
@@ -754,7 +739,7 @@ module Cnvrg
       begin
         url = URI.parse('http://169.254.169.254/latest/meta-data/spot/termination-time')
         req = Net::HTTP::Get.new(url.to_s)
-        res = Net::HTTP.start(url.host, url.port) {|http|
+        res = Net::HTTP.start(url.host, url.port) { |http|
           http.request(req)
         }
         unless res.body.include? "404"
@@ -776,7 +761,7 @@ module Cnvrg
     end
 
     def send_restart_request(job_id: nil, job_type: nil, ma_id: nil)
-      Cnvrg::API.request("#{base_resource}/spot_restart", 'POST', {job_type: job_type, job_id: job_id, machine_activity: ma_id})
+      Cnvrg::API.request("#{base_resource}/spot_restart", 'POST', { job_type: job_type, job_id: job_id, machine_activity: ma_id })
     end
 
     def get_machine_activity
@@ -796,7 +781,7 @@ module Cnvrg
       if job_type.blank? or job_id.blank?
         raise StandardError.new("Cant find job env variables")
       end
-      Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/set_pod_restart", "POST", {job_type: job_type, job_id: job_id})
+      Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/set_pod_restart", "POST", { job_type: job_type, job_id: job_id })
     end
 
     def check_job_pod_restart(job_type: nil, job_id: nil)
@@ -805,7 +790,7 @@ module Cnvrg
       if job_type.blank? or job_id.blank?
         raise StandardError.new("Cant find job env variables")
       end
-      resp = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/check_pod_restart", "GET", {job_type: job_type, job_id: job_id})
+      resp = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/check_pod_restart", "GET", { job_type: job_type, job_id: job_id })
       return [false, false] if resp.blank?
       Cnvrg::Logger.log_info("Checked for pod restart got response #{resp}")
       [resp['project_downloaded'], resp['dataset_downloaded']]
@@ -817,7 +802,7 @@ module Cnvrg
       if job_type.blank? or job_id.blank?
         raise StandardError.new("Cant find job env variables")
       end
-      Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/pre_pod_restart", "POST", {job_type: job_type, job_id: job_id})
+      Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/pre_pod_restart", "POST", { job_type: job_type, job_id: job_id })
     end
 
     def set_job_started
@@ -826,7 +811,7 @@ module Cnvrg
       if job_type.blank? or job_id.blank?
         raise StandardError.new("Cant find job env variables")
       end
-      Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/set_started", "POST", {job_type: job_type, job_id: job_id})
+      Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/set_started", "POST", { job_type: job_type, job_id: job_id })
     end
 
     def self.stop_if_project_present(project_home, project_name, owner)

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '2.1.12'
+  VERSION = '2.1.14'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 2.1.12
+  version: 2.1.14
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-30 00:00:00.000000000 Z
+date: 2023-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -434,6 +434,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.13.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.13.10
 description: A CLI tool for interacting with cnvrg.io.
 email:
 - info@cnvrg.io