cnvrg 1.6.33 → 1.6.35

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 465c50d73c84ef9a768c647926ab576225b5c4344db62b815f64685340b29e38
-  data.tar.gz: 6bf6610502b0b43c94d9cf5036b509e39da10210dc1cf6eacadb4dce1c837f60
+  metadata.gz: 5f90f59c9a970e96f5ac718fa295cc2ce4f81ff143d8d9f8848b039a4ecdd004
+  data.tar.gz: 66beb8643ccbe900863b674b20f6138d69380cd0719deba77b15ce836c14d3dd
 SHA512:
-  metadata.gz: 06fc439571eb477075c79d816b16409eb33b3171d6ebb710cce394e44ab5c37ca0da02d370a29c2957169c6a779f37b29415d8ee7af576cd7a6cf975de11dfe0
-  data.tar.gz: 5c7f72eff0112c158b3048fde389a75108151f9b527e017782c73229434180f280308d2e85d52eb765f4cba356232bcd96844aaafed0335d11f455aa73ba486b
+  metadata.gz: 771c2fc62849c1d33363d49aa555ffd7408ef3f821dc3b3106c5c94004ab1076f4d96574b12af92b80f5a7fe8ca38270875cd06801edd26145d6c7eaef249266
+  data.tar.gz: 53fd6f5f3653ed67777022e9c50f9b042f2a4b35739e23a28ff4777c5480c62e227b62633cd7b34ff44bf19a8d1ed2268555194a5650f2c103578960f2d0bf5f

data/lib/cnvrg/cli.rb

@@ -1184,7 +1184,7 @@ module Cnvrg
     end
 
     desc '', '', :hide => true
-    def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000)
+    def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, message: nil)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -1204,7 +1204,7 @@ module Cnvrg
         log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
         number_of_chunks = (@files.size.to_f / chunk_size).ceil
         if commit.blank?
-          response = @datafiles.start_commit(false, true, chunks: number_of_chunks)
+          response = @datafiles.start_commit(false, true, chunks: number_of_chunks, message: message )
           unless response #means we failed in the start commit.
             raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
           end
@@ -1837,8 +1837,6 @@ module Cnvrg
           end
           clone_resp = Project.clone_dir(slug, owner, project_name,git)
           project_home = Dir.pwd + "/" + project_name
-
-
         end
 
         if clone_resp

data/lib/cnvrg/data.rb

@@ -186,11 +186,14 @@ module Cnvrg
     desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
     method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
     method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
+    method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
     def put(dataset_url, *files)
       cli = Cnvrg::CLI.new()
       dir = options[:dir]
       commit = options[:commit]
-      cli.data_put(dataset_url, files: files, dir: dir, commit: commit)
+      message = options[:message]
+
+      cli.data_put(dataset_url, files: files, dir: dir, commit: commit, message: message)
     end
 
     desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'

data/lib/cnvrg/datafiles.rb

@@ -20,6 +20,15 @@ module Cnvrg
       @dataset = dataset
       @base_resource = "users/#{owner}/datasets/#{dataset_slug}/"
       @downloader = @dataset.get_storage_client
+      @token_issue_time = Time.current
+    end
+
+    def refresh_storage_token
+      current_time = Time.current
+      if current_time - @token_issue_time > 3.hours
+        @downloader = @dataset.get_storage_client
+        @token_issue_time = Time.current
+      end
     end
 
     def check_file_sha1(filename, org_sha1, tag: 'conflict')
@@ -117,6 +126,7 @@ module Cnvrg
     def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, prefix: '', partial_commit: nil, total: nil)
       begin
         Cnvrg::Logger.log_info("Sending Upload Files request")
+        refresh_storage_token
         error = nil
         upload_resp = nil
         10.times do
@@ -1007,6 +1017,7 @@ module Cnvrg
 
     def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false)
       begin
+        refresh_storage_token
         parallel_options = {
             in_threads: ParallelThreads,
             isolation: true

data/lib/cnvrg/files.rb

@@ -27,8 +27,16 @@ module Cnvrg
       @custom_progess = false
       @cli = cli
       @options = options
+      @token_issue_time = Time.current
     end
 
+    def refresh_storage_token
+      current_time = Time.current
+      if current_time - @token_issue_time > 3.hours
+        @client = @project.get_storage_client
+        @token_issue_time = Time.current
+      end
+    end
 
     def self.valid_file_name?(fullpath)
       VALID_FILE_NAME.match(fullpath).blank?
@@ -97,7 +105,13 @@ module Cnvrg
       files = res['files']
 
       #upload files
+      token_mutex = Mutex.new
       blob_ids = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
+
+        token_mutex.synchronize {
+          refresh_storage_token
+        }
+
         begin
           Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
         rescue => e
@@ -721,8 +735,15 @@ module Cnvrg
             in_threads: Cnvrg::Helpers.parallel_threads,
             isolation: true
         }
+
+        token_mutex = Mutex.new
+
         Parallel.map(files["keys"], parallel_options) do |f|
 
+          token_mutex.synchronize {
+            refresh_storage_token
+          }
+
           file_path = f["name"]
           if file_path.end_with? "/"
             # dir

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.6.33'
+  VERSION = '1.6.35'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.6.33
+  version: 1.6.35
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-03 00:00:00.000000000 Z
+date: 2020-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -477,7 +477,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.