cnvrg 1.9.5 → 1.9.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e708ef034df38ed0b4f5c1ac4bb02fa79a26c93b188f571256f75dbc9d2eaaa6
-  data.tar.gz: 6badf54b65660776e63c02c7d3c5dbbab83d0e1e83f6e877b48d77fad5ba3036
+  metadata.gz: 0616d28375b48a9b06dc81dd02ea8b874d864667eb58128d97f29aa8c4736153
+  data.tar.gz: 5cb79a6f2ee7c6bca1c6c812ac588e4ac2417c7f1059029037cb1bc522209462
 SHA512:
-  metadata.gz: 21d89ec4fb99c4102bc1e8e0e50df516339a1c9e9660ee8f0dd8acf3ae30bd27067f5ea4fe979de3b737bd6f748ced98f023100487a4226b7f21eed17975142c
-  data.tar.gz: 91fb2d10994c11e9b28ef3bbc128f847ac2efd641892c29ec1ec2b16d4b125266e85a6166153b66ab9e9e1c475190f6eca771e42d739a02c1136dbe8cb6c3abb
+  metadata.gz: c2094c716ebe523291de7a87d2094266c2f00cf65fc8067114b12d7739ad764553055ba85125c39a7d2571761fffa31ab7cdb057f77da5d6c20ad1a5601128d5
+  data.tar.gz: c6b9ff106b6b48c96e010da2adf00aa4a17538ee93f901bce357d805f10aedc0c58b47ce4226b3bc9a9683b8272691d5868d81779422253d703beead9b440e7b

data/cnvrg.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
     spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
     spec.add_runtime_dependency 'thor', '~> 0.19.0','>=0.19.1'
-    spec.add_runtime_dependency 'aws-sdk', '~> 3.0'
+    spec.add_runtime_dependency 'aws-sdk-s3', '~> 1'
     spec.add_runtime_dependency 'signet', '~> 0.11.0'
     spec.add_runtime_dependency 'google-cloud-env', '~> 1.2.1'
     spec.add_runtime_dependency 'google-cloud-core', '~> 1.3.2'

data/lib/cnvrg/cli.rb

@@ -4584,7 +4584,7 @@ module Cnvrg
 
     end
 
-    desc 'file_exists', description: '', hide: true
+    desc 'file_exists', '', :hide => true
     def file_exists(file)
       exit(0) if File.exists? file
       exit(1)

data/lib/cnvrg/cli/library_cli.rb

@@ -1,7 +1,7 @@
 module Cnvrg
   class LibraryCli < SubCommandBase
 
-    desc "library import", description: ""
+    desc "library import", ''
     def import
       unless File.exists? "library.yml"
         Cnvrg::CLI.log_message("Can't find library.yml", 'red')

data/lib/cnvrg/datafiles.rb

@@ -1,5 +1,5 @@
 require 'mimemagic'
-require 'aws-sdk'
+require 'aws-sdk-s3'
 require 'URLcrypt'
 require 'parallel'
 require 'fileutils'
@@ -330,7 +330,7 @@ module Cnvrg
 
     end
 
-    def upload_multiple_files_optimized(files, commit_sha1, threads: ParallelThreads, chunk_size: 1000, force: false, new_branch: false, prefix: '', partial_commit: nil)
+    def upload_multiple_files_optimized(files, commit_sha1, threads: 15, chunk_size: 1000, force: false, new_branch: false, prefix: '', partial_commit: nil)
       cli = CLI.new
       cli.log_message("Using #{threads} threads with chunk size of #{chunk_size}.", Thor::Shell::Color::GREEN)
 
@@ -406,7 +406,7 @@ module Cnvrg
         # Fetch the required files from the server:
         Parallel.map((file_chunks), in_threads: 10) do |files_chunk|
 
-          tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix)
+          tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads)
           results = request_upload_files(commit_sha1, tree, force, new_branch, partial_commit)
 
           # puts "Got #{results['files'].size} files to upload from #{files_chunk.size} files"

data/lib/cnvrg/dataset.rb

@@ -541,7 +541,7 @@ module Cnvrg
       "#{url}/#{self.owner}/datasets/#{self.slug}"
     end
 
-    def generate_chunked_idx(list_files = [], threads: IDXParallelThreads, prefix: '')
+    def generate_chunked_idx(list_files = [], threads: 15, prefix: '')
       tree = {}
       Parallel.map(list_files, in_threads: threads) do |file|
         #check if prefix exists do prefix/path otherwise path

data/lib/cnvrg/files.rb

@@ -1,5 +1,5 @@
 require 'mimemagic'
-require 'aws-sdk'
+require 'aws-sdk-s3'
 require 'URLcrypt'
 require 'tempfile'
 require 'net/http'
@@ -727,7 +727,6 @@ module Cnvrg
       unless Cnvrg::CLI.is_response_success(res, false)
         raise SignalException.new("Cant download files from the server.")
       end
-      puts res
       self.download_multpile_files_s3(res['result'], @project_home, postfix: postfix, progress: progress)
     end
 

data/lib/cnvrg/helpers/executer.rb

@@ -89,7 +89,7 @@ class Cnvrg::Helpers::Executer
   end
 
   def current_homedir
-    `env | grep HOME`.strip.split("=").try(:last)
+    `env | grep -w HOME`.strip.split("=").try(:last)
   end
 
   def spark_path
@@ -97,7 +97,7 @@ class Cnvrg::Helpers::Executer
   end
 
   def slave_homedir()
-    run_in_slave("env | grep HOME").split("=").try(:last)
+    run_in_slave("env | grep -w HOME").split("=").try(:last)
   end
 
   def slave_env

data/lib/cnvrg/job_cli.rb

@@ -1,7 +1,7 @@
 module Cnvrg
     class JobCli < SubCommandBase
 
-        desc 'log', description: '', hide: true
+        desc 'log', '', :hide => true
         method_option :level, :type => :string, :aliases => ["-l", "--level"], :default => 'info'
         method_option :step, :type => :string, :aliases => ["-s", "--step"], :default => nil
         method_option :restart, :type => :boolean, :aliases => ["-r", "--restart"], :default => false
@@ -15,7 +15,7 @@ module Cnvrg
             end
         end
 
-        desc 'requirements', description: '', hide: true
+        desc 'requirements', '', :hide => true
         def requirements
           cli = Cnvrg::CLI.new
           cli.log_start(__method__, args, options)
@@ -23,7 +23,7 @@ module Cnvrg
           @project = Project.new(project_dir, owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
         end
 
-        desc 'install_reqs', description: 'Install requirements', hide: true
+        desc 'install_reqs', 'Install requirements', :hide => true
         def install_reqs
             cli = Cnvrg::CLI.new
             cli.log_start(__method__, args, options)
@@ -33,8 +33,7 @@ module Cnvrg
             @executer.execute_cmds(commands)
         end
 
-
-        desc 'start', description: "Job Start!", hide: true
+        desc 'start', 'Job Start!', :hide => true
         def start
             cli = Cnvrg::CLI.new
             cli.log_start(__method__, args, options)
@@ -47,7 +46,7 @@ module Cnvrg
             @executer.main_thread
         end
 
-        desc 'stats', description: 'stats of agent and slave', hide: true
+        desc 'stats', 'stats of agent and slave', :hide => true
         def stats
             cli = Cnvrg::CLI.new
             cli.log_start(__method__, args, options)
@@ -59,7 +58,7 @@ module Cnvrg
             @executer.executer_stats
         end
 
-        desc 'pre_pod_stop', description: 'test', hide: true
+        desc 'pre_pod_stop', '', :hide => true
         def pre_pod_stop
             cli = Cnvrg::CLI.new
             cli.log_start(__method__, args, options)

data/lib/cnvrg/job_ssh.rb

@@ -1,6 +1,7 @@
 module Cnvrg
   class JobSsh < SubCommandBase
-    desc 'ssh start', description: 'job_id', hide: false
+
+    desc 'ssh start', 'stats of agent and slave'
     method_option :port, :type => :numeric, :aliases => ["-p", "--port"], :desc => "Port to bind into", :default => 2222
     method_option :username, :type => :string, :aliases => ["-u", "--username"], :desc => "Job container user name", :default => nil
     method_option :password, :type => :string, :aliases => ["--password"], :desc =>"Job Conatainer user name, will be set by cnvrg", :default => nil

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.9.5'
+  VERSION = '1.9.9.2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.9.5
+  version: 1.9.9.2
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-09 00:00:00.000000000 Z
+date: 2020-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -205,19 +205,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.19.1
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: signet
   requirement: !ruby/object:Gem::Requirement
@@ -453,7 +453,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.4
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.