cnvrg 2.1.3 → 2.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa682e9cf25c1d37b533721888ded60b3ebc397717fc904dcd7703a592dd8853
-  data.tar.gz: bba09312bac1e44ec8b64358dead4036e173832cb605e56803bc368b8eff6dbc
+  metadata.gz: 414c34615497d519b8df3fc1adf597aa18bb8d05eb7466b312bbfd517f0e74aa
+  data.tar.gz: ddde064e3ef2f144948d1b6d1859f4daef95290dd13ae28b544569f6b16f5be0
 SHA512:
-  metadata.gz: d8144e807411faa6c507c037309abd5e4919b317253147b6bb1eec8eaab2614ad84d32823bfc9dde405f881cefe4be9d975dd2dc21e47392c8416ddd0ed6c253
-  data.tar.gz: 46aebd82a4901774d605f40c26aee7607a0a3c407ddf1c3cc4f91625f678e5f2f0bcb8e53c92b3c5812150a6b4b98f491e3c21adace85196d82314e35ef7646e
+  metadata.gz: 8d72d7b22df91d154e0313f24511ae9f6f007c077bf0f49a9329f0546ec9983833920fd567e4abfe7ab876ae2da4aae888d77840d10d7945f956d0b8e77047c1
+  data.tar.gz: f72cfb009f744ede0fdaec505610e32fa50ee570415cd729678c2d791b79351a891569f5ef1911920ddca242bc0354c61e6aa505252793b3345cdfdaeebe0dbc

data/Readme.md

@@ -88,4 +88,24 @@
 * DEV-13815 - Bug: CLI - remove "cnvrg data sync" command
 ## Version v2.1.3
 2022-05-16
-* DEV-13981 - Bug: CLI - dataset query clone stuck at 50% then "Killed"
+* DEV-13981 - Bug: CLI - dataset query clone stuck at 50% then "Killed"
+## Version v2.1.4
+2022-05-22
+* DEV-14182 - Bug: Cli - hide 'data upload' command
+## Version v2.1.5
+2022-07-31
+* DEV-14244 - Bug: CLI - "failed to upload ongoing stats" due to NaN in float
+* DEV-14633 - Bug: End sync did not complete, causing the experiment to get stuck in "terminating"
+## Version v2.1.6
+2022-08-09
+* DEV-14682 - Bug: git-Walki: CLI/SDK experiments goes into debug mode for Github+SSH integrated projects
+## Version v2.1.7
+2022-08-24
+* DEV-15229 - Bug: CLI - warnings from faraday gem when running any CLI command
+## Version v2.1.8
+2022-09-04
+* DEV-15473 - Bug: CLI - Errors while cloning a project that has file containing spaces in their names 
+## Version v2.1.9
+2022-09-06
+* DEV-15423 - Bug: Workspace - Jupyter process gets killed
+* DEV-15451 - Bug: CLI - sync error "undefined method `encode' for nil:NilClass" on GCP storage

data/cnvrg.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'ffi', '~> 1.9', '>= 1.9.10'
     spec.add_runtime_dependency 'mimemagic', '~> 0.3.1', '>=0.3.7'
     spec.add_runtime_dependency 'faraday', '~> 0.15.2'
+    spec.add_runtime_dependency 'warning', '~> 1.3.0'
     spec.add_runtime_dependency 'netrc', '~> 0.11.0'
     spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
     spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'

data/lib/cnvrg/cli.rb

@@ -1848,7 +1848,7 @@ module Cnvrg
       log_start(__method__, args, options)
       project_home = Dir.pwd
       soft = options["soft"] || false
-      Project.stop_if_project_present(project_home, slug) if soft
+      Project.stop_if_project_present(project_home, slug, owner) if soft
       clone_resp = Project.clone_dir_remote(slug, owner, slug,true)
       exit 1 if not clone_resp
       idx_status = Project.new(get_project_home).generate_idx(files:[])
@@ -1918,7 +1918,7 @@ module Cnvrg
         clone_resp = false
         project_home = Dir.pwd
 
-        Project.stop_if_project_present(project_home, project_name) if soft
+        Project.stop_if_project_present(project_home, project_name, owner) if soft
 
         if remote and !git
           clone_resp = Project.clone_dir_remote(slug, owner, project_name,git)
@@ -2088,6 +2088,8 @@ module Cnvrg
     method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
 
     def upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:nil, total_deleted: 0, total_downloaded: 0)
+      log_message("This method is deprecated, please use 'data put' instead. for more info visit our docs: https://app.cnvrg.io/docs/cli/install.html#upload-files-to-a-dataset", Thor::Shell::Color::BLUE, !options["verbose"])
+      return
       begin
         commit, files_list = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size, :message => message
         files_to_upload, upload_errors = invoke :upload_data_files,[commit, files_list: files_list],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
@@ -4731,8 +4733,14 @@ module Cnvrg
                 end
               end
             else
-              timestamp, value = data_result&.first&.dig('value')
-              stat_value = value.present? ? ("%.2f" % value) : 0 # converting 34.685929244444445 to 34.69
+              begin
+                timestamp, value = data_result&.first&.dig('value')
+                stat_value = value.present? ? ("%.2f" % value) : 0 # converting 34.685929244444445 to 34.69
+              rescue => e
+                Cnvrg::Logger.log_info("Failed converting string into float with error: #{e.message}")
+                Cnvrg::Logger.log_error(e)
+                stat_value = 0
+              end
               stat_value = stat_value.to_i == stat_value.to_f ? stat_value.to_i : stat_value.to_f # converting 34.00 to 34
               if query_name.include? 'block'
                 stats['block_io'] = {} if stats['block_io'].blank?

data/lib/cnvrg/data.rb

@@ -81,7 +81,7 @@ module Cnvrg
       end
     end
 
-    desc "data upload", "Upload files from local dataset directory to remote server"
+    desc "data upload", "Upload files from local dataset directory to remote server", :hide => true
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false

data/lib/cnvrg/downloader/clients/azure_client.rb

@@ -15,15 +15,13 @@ module Cnvrg
 
         def download(storage_path, local_path, decrypt: true)
           prepare_download(local_path)
-
           storage_path = Cnvrg::Helpers.decrypt(@key, @iv, storage_path) if decrypt
 
           # We generate a temp uri in order to stream the file instead of using "get_blob" that overflows memory
           uri = client.send(:blob_uri, @container, storage_path)
 
-          generator = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(@account_name, @access_key)
 
-          expiring_url = generator.signed_uri(
+          expiring_url = self.signed_uri_custom(
             uri,
             false,
             service: 'b',
@@ -32,12 +30,32 @@ module Cnvrg
             start: (Time.now - (5 * 60)).utc.iso8601, # start 5 minutes ago
             expiry: (Time.now + 60 * 60 * 2).utc.iso8601 # expire in 2 hours
           )
-
           # Stream the file without loading it all into memory
           download = open(expiring_url)
           IO.copy_stream(download, local_path)
         end
 
+        def signed_uri_custom(uri, use_account_sas, options)
+          # url sent to generate_service_sas_token should be DECODED (file names with spaces should not be encoded with %20)
+          url = URI.decode(uri.path)
+          generator = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(@account_name, @access_key)
+
+          CGI::parse(uri.query || "").inject({}) { |memo, (k, v)| memo[k.to_sym] = v; memo }
+
+          if options[:service] == (nil) && uri.host != (nil)
+            host_splits = uri.host.split(".")
+            options[:service] = host_splits[1].chr if host_splits.length > 1 && host_splits[0] == @account_name
+          end
+
+          sas_params = if use_account_sas
+                         generator.generate_account_sas_token(options)
+                       else
+                         generator.generate_service_sas_token(url, options)
+                       end
+
+          URI.parse(uri.to_s + (uri.query.nil? ? "?" : "&") + sas_params)
+        end
+
         def upload(storage_path, local_path)
           begin
             client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))

data/lib/cnvrg/downloader/clients/gcp_client.rb

@@ -6,7 +6,7 @@ module Cnvrg
       class GcpClient < Client
         def initialize(project_id: nil, credentials: nil, bucket_name: nil, sts: nil)
           @key, @iv = extract_key_iv(sts)
-          @project_id = Cnvrg::Helpers.decrypt(@key, @iv, project_id)
+          @project_id = Cnvrg::Helpers.decrypt(@key, @iv, project_id) if project_id.present?
           @credentials_path = Cnvrg::Helpers.decrypt(@key, @iv, credentials)
           @tempfile = nil
           @bucket_name = Cnvrg::Helpers.decrypt(@key, @iv, bucket_name)

data/lib/cnvrg/helpers/agent.rb

@@ -22,6 +22,7 @@ class Cnvrg::Helpers::Agent
     @slug = slug
     @files_exist = files_exist
     @container_name = container_name
+    @execute_command_completed = false
     @is_new_main = !ENV["MAIN_CONTAINER_PORT"].blank?
     @main_name = @is_new_main ? "main" : "slave"
     @run_in_main = @container_name.downcase == @main_name
@@ -79,6 +80,7 @@ class Cnvrg::Helpers::Agent
       command_status = Status::ABORTED
       @exit_status = 127
     end
+    @execute_command_completed = true
     finish_log = "Command: #{@command} with slug: #{@slug} finished"
     finish_log += " after #{@real_execution_retries} retries" if @real_execution_retries > 0
     log_internal(finish_log)
@@ -97,7 +99,7 @@ class Cnvrg::Helpers::Agent
 
   def periodic_thread
     Thread.new do
-        while @exit_status.blank?
+        while !@execute_command_completed
           Thread.exit if @log_interval.blank?
           sleep(@log_interval)
           send_logs
@@ -179,11 +181,17 @@ class Cnvrg::Helpers::Agent
       end
     end
     @exit_status = $?.exitstatus
+  rescue NoMethodError => e
+    log_internal("No Method Error: #{e}", level: LogLevel::ERROR)
+    @exit_status = 129
   rescue Timeout::Error
     Process.kill(0, @pid)
     @errors << {log: "Command timed out!", timestamp: Time.now}
     log_internal("Command timed out!", level: LogLevel::ERROR)
     @exit_status = 124
+  rescue => e
+    log_internal("Error: #{e}", level: LogLevel::ERROR)
+    @exit_status = 129
   ensure
     retry_command if @retries != 0 and @exit_status !=0
     @exit_status

data/lib/cnvrg/project.rb

@@ -829,12 +829,12 @@ module Cnvrg
       Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/set_started", "POST", {job_type: job_type, job_id: job_id})
     end
 
-    def self.stop_if_project_present(project_home, project_name)
+    def self.stop_if_project_present(project_home, project_name, owner)
       cli = Cnvrg::CLI.new()
       config = YAML.load_file(project_home + "/.cnvrg/config.yml")
       local_commit = YAML.load_file(project_home + "/.cnvrg/idx.yml")[:commit] rescue nil
       return if local_commit.blank?
-      if config[:project_name] == project_name
+      if config[:project_name] == project_name && config[:owner] == owner
         cli.log_message("Project already present, clone aborted")
         exit(0)
       end

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '2.1.3'
+  VERSION = '2.1.9'
 end

data/lib/cnvrg.rb

@@ -1,3 +1,7 @@
+require 'warning'
+Warning.ignore(/.*::CompositeIO is deprecated.*/)
+Warning.ignore(/.*::Parts is deprecated.*/)
+Warning.ignore(/.*::UploadIO is deprecated.*/)
 require 'rubygems'
 require 'cnvrg/version'
 require 'cnvrg/cli'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 2.1.3
+  version: 2.1.9
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-05-16 00:00:00.000000000 Z
+date: 2022-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -150,6 +150,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.15.2
+- !ruby/object:Gem::Dependency
+  name: warning
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.0
 - !ruby/object:Gem::Dependency
   name: netrc
   requirement: !ruby/object:Gem::Requirement