cnvrg 1.9.8 → 1.9.9.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d2e037c6264223158a1d85eb3c570453d0c67982e1088060a439bfcc7ffac37f
-  data.tar.gz: e71b3030503d3f128ba912dcb798faa8626ebad4dd0377b94eb6b3ea74df9c5e
+  metadata.gz: 8d2e4f14c9dfbb2875eebd2de45262eff3e4a2bc848e7d0305610092c84faa46
+  data.tar.gz: 6ba88a0190689ba218441f1f6c6d9307b71cf5a889a3649692aa556656e533ab
 SHA512:
-  metadata.gz: e8b35e3e285a0ee031f4c61680004b91db9be7387c3544f1386eb92f7961ea0b7ef7191594255ef9cd4f3531119db86da682557a7380de2e36973173fc4749ac
-  data.tar.gz: f5be3adefda82fa9d37a59055b341b894551ea87e6b65045ae4e3fcec4db65f7d9a02d632e43116de046bc2fd6d17636351bb7af1ee0f3ee2593dbd4890bbc1c
+  metadata.gz: cd22dd71906534018350166af1f23d20b8aa085dfb88b09d750974dd0f0229337a36efcdd34455ad9e0435846e7f4333bd5c4a5e3cf353ad8ca926ee011a8993
+  data.tar.gz: a4cae273a0000073d3955f60c0ebc5dd88c5b4ac1f28d5c09b1e99bbffe284446d8bee93bd97d33f2ba58052222d51c69c28a2c4bc31322205cf779e4adf7f2f

data/lib/cnvrg/cli.rb

@@ -1191,7 +1191,7 @@ module Cnvrg
     end
 
     desc '', '', :hide => true
-    def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, force: false, threads: 15, message: nil)
+    def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, force: false, override: false, threads: 15, message: nil)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -1207,7 +1207,7 @@ module Cnvrg
         log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
         number_of_chunks = (@files.size.to_f / chunk_size).ceil
         if commit.blank?
-          response = @datafiles.start_commit(false, true, chunks: number_of_chunks, message: message )
+          response = @datafiles.start_commit(false, force, chunks: number_of_chunks, message: message )
           unless response #means we failed in the start commit.
             raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
           end
@@ -1229,7 +1229,7 @@ module Cnvrg
         @datafiles.upload_multiple_files_optimized(
           @files,
           @commit,
-          force: force,
+          override: override,
           chunk_size: chunk_size,
           prefix: dir,
           threads: threads
@@ -1241,7 +1241,7 @@ module Cnvrg
           raise SignalException.new(1, res.msg)
         end
 
-        res = @datafiles.end_commit(@commit,false, success: true, commit_type: "put")
+        res = @datafiles.end_commit(@commit,force, success: true, commit_type: "put")
         msg = res['result']
         response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
         unless response.is_success?
@@ -4584,7 +4584,7 @@ module Cnvrg
 
     end
 
-    desc 'file_exists', description: '', hide: true
+    desc 'file_exists', '', :hide => true
     def file_exists(file)
       exit(0) if File.exists? file
       exit(1)

data/lib/cnvrg/cli/library_cli.rb

@@ -1,7 +1,7 @@
 module Cnvrg
   class LibraryCli < SubCommandBase
 
-    desc "library import", description: ""
+    desc "library import", ''
     def import
       unless File.exists? "library.yml"
         Cnvrg::CLI.log_message("Can't find library.yml", 'red')

data/lib/cnvrg/data.rb

@@ -213,13 +213,16 @@ module Cnvrg
     method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
     method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
+    method_option :override, :type => :boolean, :aliases => ["--override"], :default => false
     method_option :threads, :type => :numeric, :aliases => ["-t","--threads"], :default => 15
     method_option :chunk_size, :type => :numeric, :aliases => ["-cs","--chunk"], :default => 1000
     method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
+
     def put(dataset_url, *files)
       cli = Cnvrg::CLI.new()
       dir = options[:dir]
-      force = options["force"]
+      force = options[:force]
+      override = options[:override]
       commit = options[:commit]
       message = options[:message]
       threads = options[:threads]
@@ -230,6 +233,7 @@ module Cnvrg
           dir: dir,
           commit: commit,
           force: force,
+          override: override,
           threads: threads,
           chunk_size: chunk_size,
           message: message

data/lib/cnvrg/datafiles.rb

@@ -302,13 +302,13 @@ module Cnvrg
       end
     end
 
-    def request_upload_files(commit_sha1, tree, force, new_branch, partial_commit)
+    def request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
       retry_count = 0
       loop do
         upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {
             commit_sha1: commit_sha1,
             tree: tree,
-            force: force,
+            override: override,
             is_branch: new_branch,
             partial_commit: partial_commit
         })
@@ -330,7 +330,7 @@ module Cnvrg
 
     end
 
-    def upload_multiple_files_optimized(files, commit_sha1, threads: ParallelThreads, chunk_size: 1000, force: false, new_branch: false, prefix: '', partial_commit: nil)
+    def upload_multiple_files_optimized(files, commit_sha1, threads: 15, chunk_size: 1000, override: false, new_branch: false, prefix: '', partial_commit: nil)
       cli = CLI.new
       cli.log_message("Using #{threads} threads with chunk size of #{chunk_size}.", Thor::Shell::Color::GREEN)
 
@@ -406,8 +406,8 @@ module Cnvrg
         # Fetch the required files from the server:
         Parallel.map((file_chunks), in_threads: 10) do |files_chunk|
 
-          tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix)
-          results = request_upload_files(commit_sha1, tree, force, new_branch, partial_commit)
+          tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads)
+          results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
 
           # puts "Got #{results['files'].size} files to upload from #{files_chunk.size} files"
 
@@ -1149,43 +1149,29 @@ module Cnvrg
     end
 
     def start_commit(new_branch, force=false, chunks: 0, dataset: @dataset, message:nil)
-      begin
-        #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
-        # create a new commit.
-        idx = (force || new_branch) ? {} : dataset.get_idx
-        commit = idx[:commit]
-        next_commit = idx[:next_commit]
-        response = Cnvrg::API.request(
-            "#{base_resource}/commit/start",
-            'POST',
-            {
-                dataset_slug: @dataset_slug,
-                new_branch: new_branch,
-                force:force,
-                username: @owner,
-                current_commit: commit,
-                next_commit: next_commit,
-                total_chunks: chunks,
-                message: message
-            }
-        )
+      #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
+      # create a new commit.
+      idx = {}
+      commit = idx[:commit]
+      next_commit = idx[:next_commit]
+      response = Cnvrg::API.request(
+          "#{base_resource}/commit/start",
+          'POST',
+          {
+              dataset_slug: @dataset_slug,
+              new_branch: new_branch,
+              force:force,
+              username: @owner,
+              current_commit: commit,
+              next_commit: next_commit,
+              total_chunks: chunks,
+              message: message
+          }
+      )
       Cnvrg::CLI.is_response_success(response, true)
-      return response
-      rescue => e
-        return false
-      end
-    end
-
-    def last_valid_commit()
-      begin
-        #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
-        # create a new commit.
-        response = Cnvrg::API.request("#{base_resource}/last_valid_commit", 'GET')
-        Cnvrg::CLI.is_response_success(response, true)
-        return response
-      rescue => e
-        return false
-      end
+      response
+    rescue => e
+      false
     end
 
     def end_commit(commit_sha1, force, success: true, uploaded_files: 0, commit_type: nil)
@@ -1319,10 +1305,10 @@ module Cnvrg
               # blob
             local_path = "#{local_path}.conflict" if conflict
             storage_path = f["path"]
-            if File.exists? local_path
-              Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
-              next
-            end
+            # if File.exists? local_path
+            #   Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
+            #   next
+            # end
             resp = @downloader.download(storage_path, local_path)
             Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
           rescue => e

data/lib/cnvrg/dataset.rb

@@ -541,7 +541,7 @@ module Cnvrg
       "#{url}/#{self.owner}/datasets/#{self.slug}"
     end
 
-    def generate_chunked_idx(list_files = [], threads: IDXParallelThreads, prefix: '')
+    def generate_chunked_idx(list_files = [], threads: 15, prefix: '')
       tree = {}
       Parallel.map(list_files, in_threads: threads) do |file|
         #check if prefix exists do prefix/path otherwise path

data/lib/cnvrg/helpers/executer.rb

@@ -89,7 +89,7 @@ class Cnvrg::Helpers::Executer
   end
 
   def current_homedir
-    `env | grep HOME`.strip.split("=").try(:last)
+    `env | grep -w HOME`.strip.split("=").try(:last)
   end
 
   def spark_path
@@ -97,7 +97,7 @@ class Cnvrg::Helpers::Executer
   end
 
   def slave_homedir()
-    run_in_slave("env | grep HOME").split("=").try(:last)
+    run_in_slave("env | grep -w HOME").split("=").try(:last)
   end
 
   def slave_env

data/lib/cnvrg/job_cli.rb

@@ -1,7 +1,7 @@
 module Cnvrg
     class JobCli < SubCommandBase
 
-        desc 'log', description: '', hide: true
+        desc 'log', '', :hide => true
         method_option :level, :type => :string, :aliases => ["-l", "--level"], :default => 'info'
         method_option :step, :type => :string, :aliases => ["-s", "--step"], :default => nil
         method_option :restart, :type => :boolean, :aliases => ["-r", "--restart"], :default => false
@@ -15,7 +15,7 @@ module Cnvrg
             end
         end
 
-        desc 'requirements', description: '', hide: true
+        desc 'requirements', '', :hide => true
         def requirements
           cli = Cnvrg::CLI.new
           cli.log_start(__method__, args, options)
@@ -23,7 +23,7 @@ module Cnvrg
           @project = Project.new(project_dir, owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
         end
 
-        desc 'install_reqs', description: 'Install requirements', hide: true
+        desc 'install_reqs', 'Install requirements', :hide => true
         def install_reqs
             cli = Cnvrg::CLI.new
             cli.log_start(__method__, args, options)
@@ -33,8 +33,7 @@ module Cnvrg
             @executer.execute_cmds(commands)
         end
 
-
-        desc 'start', description: "Job Start!", hide: true
+        desc 'start', 'Job Start!', :hide => true
         def start
             cli = Cnvrg::CLI.new
             cli.log_start(__method__, args, options)
@@ -47,7 +46,7 @@ module Cnvrg
             @executer.main_thread
         end
 
-        desc 'stats', description: 'stats of agent and slave', hide: true
+        desc 'stats', 'stats of agent and slave', :hide => true
         def stats
             cli = Cnvrg::CLI.new
             cli.log_start(__method__, args, options)
@@ -59,7 +58,7 @@ module Cnvrg
             @executer.executer_stats
         end
 
-        desc 'pre_pod_stop', description: 'test', hide: true
+        desc 'pre_pod_stop', '', :hide => true
         def pre_pod_stop
             cli = Cnvrg::CLI.new
             cli.log_start(__method__, args, options)

data/lib/cnvrg/job_ssh.rb

@@ -1,6 +1,7 @@
 module Cnvrg
   class JobSsh < SubCommandBase
-    desc 'ssh start', description: 'job_id', hide: false
+
+    desc 'ssh start', 'stats of agent and slave'
     method_option :port, :type => :numeric, :aliases => ["-p", "--port"], :desc => "Port to bind into", :default => 2222
     method_option :username, :type => :string, :aliases => ["-u", "--username"], :desc => "Job container user name", :default => nil
     method_option :password, :type => :string, :aliases => ["--password"], :desc =>"Job Conatainer user name, will be set by cnvrg", :default => nil

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.9.8'
+  VERSION = '1.9.9.5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.9.8
+  version: 1.9.9.5
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-09 00:00:00.000000000 Z
+date: 2020-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -453,7 +453,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.4
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.