cnvrg 2.1.6 → 2.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 296eba8c8dab87e1a16b7980c1e80b013be95af52cfc184a12cf366f676a3a2b
-  data.tar.gz: '08ad62abd898bb6bb1a9099237f5ebd854f87359ecc571036ce351b259127d78'
+  metadata.gz: 850c099310fb739283b4fe34c0c0eccfb9bd4297e9d525daa497eb45481ea707
+  data.tar.gz: d84a2854fe5ede6ab5a65804d11068a00c88dccf5d341a9d54728c3f5870a9b0
 SHA512:
-  metadata.gz: f5174c705ed765c76538401ea14ea2c14e267d98836b3d8e8b66a72d31d4f7ee2eeb313fbd080cd15a513bb4149f907aef075be2d7f6bab4b96afefb4ba5f341
-  data.tar.gz: 355b440e9a009571e2f097599d14418fcdbe0a79cfaae8473637722745923f8b138b4fdbc0e3ee60107122835f98468811661b770ffee4008f8b141250361fce
+  metadata.gz: 0e3da1a344367dacf8496c0e59de07770ce69db0ab59009d14fd27bc15ae6e9e56bb653e49c95d732a5b1dde962a550cba8da3310806c615015fa693ae9899d3
+  data.tar.gz: 01eda251528f076aa244558272ef4a630145fb7e8752f5f25b7c1f6bfb39be61e33189ad1e8b5e50ac1d80825de303ec479186eb65b0db6d2b62cce1b693c99a

data/Readme.md

@@ -98,4 +98,23 @@
 * DEV-14633 - Bug: End sync did not complete, causing the experiment to get stuck in "terminating"
 ## Version v2.1.6
 2022-08-09
-* DEV-14682 - Bug: git-Walki: CLI/SDK experiments goes into debug mode for Github+SSH integrated projects
+* DEV-14682 - Bug: git-Walki: CLI/SDK experiments goes into debug mode for Github+SSH integrated projects
+## Version v2.1.7
+2022-08-24
+* DEV-15229 - Bug: CLI - warnings from faraday gem when running any CLI command
+## Version v2.1.8
+2022-09-04
+* DEV-15473 - Bug: CLI - Errors while cloning a project that has file containing spaces in their names 
+## Version v2.1.9
+2022-09-06
+* DEV-15423 - Bug: Workspace - Jupyter process gets killed
+* DEV-15451 - Bug: CLI - sync error "undefined method `encode' for nil:NilClass" on GCP storage
+## Version v2.1.10
+2022-10-26
+* DEV-15858 - Bug: Tensorboard compare - ongoing experiments' artifacts aren't cloned to session
+## Version v2.1.11
+2022-11-03
+* DEV-16090 - Bug: Tensorboard compare - webapp gets stuck in "init" state 
+## Version v2.1.12
+2022-11-09
+* DEV-15972 - Bug: Customer cannot use dataset query due to ssl error

data/cnvrg.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'ffi', '~> 1.9', '>= 1.9.10'
     spec.add_runtime_dependency 'mimemagic', '~> 0.3.1', '>=0.3.7'
     spec.add_runtime_dependency 'faraday', '~> 0.15.2'
+    spec.add_runtime_dependency 'warning', '~> 1.3.0'
     spec.add_runtime_dependency 'netrc', '~> 0.11.0'
     spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
     spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'

data/lib/cnvrg/cli.rb

@@ -1019,7 +1019,7 @@ module Cnvrg
               begin
                 unless File.exist?(fullpath)
                   downloader.safe_operation("#{abs_path}/#{file_name}") do
-                    download = open(f["url"])
+                    download = open(f["url"],{ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE})
                     IO.copy_stream(download, fullpath)
                   end
                 end
@@ -5003,32 +5003,28 @@ module Cnvrg
 
       log_message("compare is running")
       while true
-        log_message("compare is running for slugs #{slugs}")
+        log_message("Comparing the following experiment slugs: #{slugs}")
         slugs.each do |exp_slug|
           begin
             if exps_map[exp_slug].blank?
               exp = @project.get_experiment(exp_slug)["experiment"]
             else
               exp = exps_map[exp_slug]
-              log_message("#{exp["title"]} has ended skipping it", Thor::Shell::Color::BLUE)
+              log_message("Experiment '#{exp["title"]}' end commit already cloned, skipping it", Thor::Shell::Color::BLUE)
               next
             end
             exp_name = exp["title"]
             if exp["end_commit"].present? and exp["status"] != "Ongoing"
-              log_message("#{exp_name} has ended, getting files from end commit", Thor::Shell::Color::BLUE)
+              log_message("Experiment '#{exp_name}' has ended, getting files from its end commit", Thor::Shell::Color::BLUE)
               num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project)
               exps_map[exp_slug] = exp
             else
-              log_message("#{exp_name} is running should get logs", Thor::Shell::Color::BLUE)
-              success, num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
-              if !success and exp["last_successful_commit"].present? and !copied_commits.include?(exp["last_successful_commit"]["sha1"])
-                log_message("Failed to get kube files, using last commit", Thor::Shell::Color::BLUE)
-                num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"]["sha1"])
-                copied_commits << exp["last_successful_commit"]["sha1"]
-              end
+              log_message("Experiment '#{exp_name}' is running, getting files from its last successful commit", Thor::Shell::Color::BLUE)
+              num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"]["sha1"])
+              copied_commits << exp["last_successful_commit"]["sha1"]
             end
 
-            log_message("New tf files copied", Thor::Shell::Color::BLUE) if num_of_new_files > 0
+            log_message("New .tfevent files downloaded", Thor::Shell::Color::BLUE) if num_of_new_files > 0
           rescue => e
             Cnvrg::Logger.log_error(e)
           end

data/lib/cnvrg/downloader/clients/azure_client.rb

@@ -15,15 +15,13 @@ module Cnvrg
 
         def download(storage_path, local_path, decrypt: true)
           prepare_download(local_path)
-
           storage_path = Cnvrg::Helpers.decrypt(@key, @iv, storage_path) if decrypt
 
           # We generate a temp uri in order to stream the file instead of using "get_blob" that overflows memory
           uri = client.send(:blob_uri, @container, storage_path)
 
-          generator = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(@account_name, @access_key)
 
-          expiring_url = generator.signed_uri(
+          expiring_url = self.signed_uri_custom(
             uri,
             false,
             service: 'b',
@@ -32,12 +30,32 @@ module Cnvrg
             start: (Time.now - (5 * 60)).utc.iso8601, # start 5 minutes ago
             expiry: (Time.now + 60 * 60 * 2).utc.iso8601 # expire in 2 hours
           )
-
           # Stream the file without loading it all into memory
           download = open(expiring_url)
           IO.copy_stream(download, local_path)
         end
 
+        def signed_uri_custom(uri, use_account_sas, options)
+          # url sent to generate_service_sas_token should be DECODED (file names with spaces should not be encoded with %20)
+          url = URI.decode(uri.path)
+          generator = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(@account_name, @access_key)
+
+          CGI::parse(uri.query || "").inject({}) { |memo, (k, v)| memo[k.to_sym] = v; memo }
+
+          if options[:service] == (nil) && uri.host != (nil)
+            host_splits = uri.host.split(".")
+            options[:service] = host_splits[1].chr if host_splits.length > 1 && host_splits[0] == @account_name
+          end
+
+          sas_params = if use_account_sas
+                         generator.generate_account_sas_token(options)
+                       else
+                         generator.generate_service_sas_token(url, options)
+                       end
+
+          URI.parse(uri.to_s + (uri.query.nil? ? "?" : "&") + sas_params)
+        end
+
         def upload(storage_path, local_path)
           begin
             client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))

data/lib/cnvrg/downloader/clients/gcp_client.rb

@@ -6,7 +6,7 @@ module Cnvrg
       class GcpClient < Client
         def initialize(project_id: nil, credentials: nil, bucket_name: nil, sts: nil)
           @key, @iv = extract_key_iv(sts)
-          @project_id = Cnvrg::Helpers.decrypt(@key, @iv, project_id)
+          @project_id = Cnvrg::Helpers.decrypt(@key, @iv, project_id) if project_id.present?
           @credentials_path = Cnvrg::Helpers.decrypt(@key, @iv, credentials)
           @tempfile = nil
           @bucket_name = Cnvrg::Helpers.decrypt(@key, @iv, bucket_name)

data/lib/cnvrg/helpers/agent.rb

@@ -22,6 +22,7 @@ class Cnvrg::Helpers::Agent
     @slug = slug
     @files_exist = files_exist
     @container_name = container_name
+    @execute_command_completed = false
     @is_new_main = !ENV["MAIN_CONTAINER_PORT"].blank?
     @main_name = @is_new_main ? "main" : "slave"
     @run_in_main = @container_name.downcase == @main_name
@@ -79,6 +80,7 @@ class Cnvrg::Helpers::Agent
       command_status = Status::ABORTED
       @exit_status = 127
     end
+    @execute_command_completed = true
     finish_log = "Command: #{@command} with slug: #{@slug} finished"
     finish_log += " after #{@real_execution_retries} retries" if @real_execution_retries > 0
     log_internal(finish_log)
@@ -97,7 +99,7 @@ class Cnvrg::Helpers::Agent
 
   def periodic_thread
     Thread.new do
-        while @exit_status.blank?
+        while !@execute_command_completed
           Thread.exit if @log_interval.blank?
           sleep(@log_interval)
           send_logs

data/lib/cnvrg/helpers.rb

@@ -374,48 +374,6 @@ parameters:
       end
       return num_of_new_files
     end
-
-    def get_experiment_events_log_via_kubectl(exp, namespace)
-      dest_dir = exp["slug"]
-      result = `kubectl -n #{namespace} get pods | grep #{exp["slug"]}`
-
-      pod_name = result.split(" ")[0]
-      return false, 0 if pod_name.blank?
-      FileUtils.mkdir_p(dest_dir)
-      working_dir = `kubectl -n #{namespace} exec #{pod_name} -c agent -- pwd`
-      working_dir.strip!
-      res = `kubectl -n #{namespace} exec #{pod_name} -c agent -- /bin/bash -c "ls -R #{working_dir}"`
-      files_and_folders = res.split("\n\n")
-      all_files = []
-
-      files_and_folders.each do |file_and_folder|
-        files = file_and_folder.split("\n")
-        if files.first.include?(":")
-          folder = files.first.gsub(":", "")
-
-          folder = folder.sub(working_dir + "/", "")
-          files = files.drop(1)
-        end
-        files.each do |file|
-          if file.include?("tfevents")
-            all_files << "#{folder}/#{file}"
-          end
-        end
-      end
-
-      num_of_new_files = 0
-      all_files.each do |file|
-        file_dir = "#{dest_dir}/#{File.dirname(file)}"
-        FileUtils.mkdir_p(file_dir)
-        num_of_new_files += 1 unless File.exist?("#{dest_dir}/#{file}")
-        res = `kubectl -n #{namespace} cp #{pod_name}:#{file} -c agent #{dest_dir}/#{file}`
-      end
-
-      return true, num_of_new_files
-    rescue => e
-      Cnvrg::Logger.log_error(e)
-      return false, 0
-    end
   end
 
 end

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '2.1.6'
+  VERSION = '2.1.12'
 end

data/lib/cnvrg.rb

@@ -1,3 +1,7 @@
+require 'warning'
+Warning.ignore(/.*::CompositeIO is deprecated.*/)
+Warning.ignore(/.*::Parts is deprecated.*/)
+Warning.ignore(/.*::UploadIO is deprecated.*/)
 require 'rubygems'
 require 'cnvrg/version'
 require 'cnvrg/cli'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 2.1.6
+  version: 2.1.12
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-08-09 00:00:00.000000000 Z
+date: 2022-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -150,6 +150,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.15.2
+- !ruby/object:Gem::Dependency
+  name: warning
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.0
 - !ruby/object:Gem::Dependency
   name: netrc
   requirement: !ruby/object:Gem::Requirement