cnvrg 1.10.12 → 1.10.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9c519291dee531a33698f1737d8019b9a9affff30c00c60ebb5904cf2ab4daa
-  data.tar.gz: 1dd54d29c1f0b7db384bfe62ae6f3678a118375515d6da451ef58094adfa334d
+  metadata.gz: 4d377618af4c931ec6553e946bb1f32c25bbb7bccd538f040309f4318c20e3bd
+  data.tar.gz: 744a798d06c8c562991be9b68cf6b8258870e657a93e2125ff3bc3c76bd7b2b6
 SHA512:
-  metadata.gz: d4ce4dea833009867b132bdf6db3fa79bcc7396e828d2f7ad05433ca5c73bf06a999eb40d75d8895f8ae4108d7d6fb30fdc3149001edd63a25f785956abb2472
-  data.tar.gz: 1082b5f3dc5da602bb2089a97dec85774bed3a43c05c9247acf64c0bff7a6d50a641dee92f8a4b8999dd3732c5d3c75eec14c6e4b578c9b09d49f5bab7452e39
+  metadata.gz: e52939b069a7ef45d69a0ae8a086291245db90602a13039c7981ac9a3c85913cf3c3b105198a149cc2913668a5fcf59288266ada60d71b5c327b3b90bc3c594a
+  data.tar.gz: 03d8880d00c707ab6afcb3509da1e02c4f01aeaf5e378b49a3ec7c651b2b00176b01be5de9320da652173480279bf1d820cf9e2a255ec1e21217e361c65fd375

data/lib/cnvrg/cli.rb

@@ -1208,12 +1208,14 @@ module Cnvrg
         log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
         number_of_chunks = (@files.size.to_f / chunk_size).ceil
         if commit.blank?
+          Cnvrg::Logger.info("Creating commit")
           response = @datafiles.start_commit(false, force, chunks: number_of_chunks, message: message )
           unless response #means we failed in the start commit.
             raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
           end
           @commit =  response['result']['commit_sha1']
         elsif commit.eql? "latest"
+          Cnvrg::Logger.info("Put files in latest commit")
           response = @datafiles.last_valid_commit()
           unless response #means we failed in the start commit.
             raise SignalException.new(1, "Cant put files into commit:#{commit}, check the dataset id and commitc")
@@ -1233,15 +1235,15 @@ module Cnvrg
           override: override,
           chunk_size: chunk_size,
           prefix: dir,
-          threads: threads
+          threads: threads,
         )
-
+        Cnvrg::Logger.info("Finished upload files")
         # This is for backwards compatibility only and should be removed in future versions:
         res = @datafiles.put_commit(@commit)
         unless res.is_success?
           raise SignalException.new(1, res.msg)
         end
-
+        Cnvrg::Logger.info("Saving commit on server")
         res = @datafiles.end_commit(@commit,force, success: true, commit_type: "put")
         msg = res['result']
         response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
@@ -5301,8 +5303,8 @@ module Cnvrg
               count += 1
             end
             if File.exist? logfile_old
-              @files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
-              @files.upload_log_file(logfile_old, "log_#{date}.log", yesterday)
+              #@files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
+              #@files.upload_log_file(logfile_old, "log_#{date}.log", yesterday)
               FileUtils.remove logfile_old
             end
 

data/lib/cnvrg/data.rb

@@ -232,15 +232,15 @@ module Cnvrg
       threads = options[:threads]
       chunk_size = options[:chunk_size]
       cli.data_put(
-          dataset_url,
-          files: files,
-          dir: dir,
-          commit: commit,
-          force: force,
-          override: override,
-          threads: threads,
-          chunk_size: chunk_size,
-          message: message
+        dataset_url,
+        files: files,
+        dir: dir,
+        commit: commit,
+        force: force,
+        override: override,
+        threads: threads,
+        chunk_size: chunk_size,
+        message: message
       )
     end
 

data/lib/cnvrg/datafiles.rb

@@ -376,16 +376,16 @@ module Cnvrg
 
               if uploaded_files.size == chunk_size or progressbar.finished?
                 refresh_storage_token
-                # puts "progress: #{progress_queue.length}"
-                # puts "files: #{file_queue.length}"
+                Cnvrg::Logger.info("Finished upload chunk of #{chunk_size} files")
                 Cnvrg::Logger.info("Sending Upload files save")
                 blob_ids = uploaded_files.map {|f| f['bv_id']}
+                Cnvrg::Logger.info("Sending chunk to server")
                 upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
                 unless Cnvrg::CLI.is_response_success(upload_resp, false)
                   Cnvrg::Logger.log_method(bind: binding)
                   raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
                 end
-                # cli.log_message("Saved file chunk to server", Thor::Shell::Color::GREEN)
+                Cnvrg::Logger.info("Chunk saved on server")
                 uploaded_files = []
                 dirs = []
               end
@@ -394,7 +394,7 @@ module Cnvrg
             end
 
             if progressbar.finished?
-              # puts "finished"
+              Cnvrg::Logger.info("Progress bar finished closing queues")
               file_queue.close()
               progress_queue.close()
               Thread.exit
@@ -405,8 +405,9 @@ module Cnvrg
         file_chunks = files.each_slice(chunk_size).to_a
         # Fetch the required files from the server:
         Parallel.map((file_chunks), in_threads: 10) do |files_chunk|
-
+          Cnvrg::Logger.info("Generating chunk idx")
           tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads)
+          Cnvrg::Logger.info("Getting files info from server")
           results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
 
           if results['files'].blank?
@@ -430,7 +431,7 @@ module Cnvrg
             file_queue.push tree[key].merge(files_to_upload[key])
           end
         end
-
+        Cnvrg::Logger.info("Finishing sub processes of datasets' upload")
         progress_thread.join()
         worker_threads.each(&:join)
 

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.10.12'
+  VERSION = '1.10.15'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.10.12
+  version: 1.10.15
 platform: ruby
 authors:
 - Yochay Ettun