cnvrg 1.11.24 → 1.11.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8a5d09f38ee4ce25caecdaf73b77e37fadff3b745454030cb15863210c1b319
-  data.tar.gz: 1f3e27bc4fa778ff048e7257f2ce753009ceaf0801c4ca3f4facfdc3011ad940
+  metadata.gz: 4fa2d41b1f7a80c14522b607d157762fccbee612e2b5663e41f102dde8ea8b50
+  data.tar.gz: 1d8acf0aaeb2e47c9d9bb424a1a7f8cba2e75cddb0dced119309aa49702f21d8
 SHA512:
-  metadata.gz: 14d79f14723eff1d8df81907183fabbe97270fd49610d4aa456fa820e4c19994c3d2bd1b7c64160f4d66a0636ff385a32b86f6fed2bedf9c0ed98c7968267c23
-  data.tar.gz: 80b4644fc6f560e54280ab726983f4e5d89978255991b29c510452b5514019025a3dcae45e4ad5f9a36e97823909eb6d1f2ee892cc63ef5090fa78ecacf595fe
+  metadata.gz: 86fa4e56ddb38cce2b6e18efdeab1316570a42bfe3f450c1eb497dba99a2444a10c2042c51ff3d5da3507c1ce22a2e7cedadd203e3514b93dd0c9c494dcab785
+  data.tar.gz: c3388652c80427fd5e8e3e89289c853f238a2a01eddd46a04263e9e6df44feb64f2c315307ccad055ba7dad6dc8dad0c6619183fb7adcc0cf1a67e36553717d5

data/lib/cnvrg/cli.rb

@@ -858,7 +858,7 @@ module Cnvrg
     method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
     method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
     method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
-    def clone_data(dataset_url, only_tree=false, commit=nil, query=nil, read=false, remote=false, flatten: false, relative: false, soft: false, threads: 15)
+    def clone_data(dataset_url, only_tree=false, commit=nil, query=nil, read=false, remote=false, flatten: false, relative: false, soft: false, threads: 15, cache_link: false)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -904,7 +904,7 @@ module Cnvrg
 
           commit = response["result"]["commit"]
           files_count = response["result"]["file_count"]
-          files = @files.get_clone_chunk(commit: commit)
+          files = @files.get_clone_chunk(commit: commit, cache_link: cache_link)
           downloaded_files = 0
           progressbar = ProgressBar.create(:title => "Download Progress",
                                            :progress_mark => '=',
@@ -917,7 +917,7 @@ module Cnvrg
 
           while files['keys'].length > 0
             Cnvrg::Logger.log_info("download multiple files, #{downloaded_files.size} files downloaded")
-            @files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read, flatten: flatten, threads: threads)
+            @files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read, flatten: flatten, threads: threads, cache_link: cache_link)
 
             downloaded_files += files['keys'].length
             files = @files.get_clone_chunk(commit: commit, latest_id: files['latest'])
@@ -1201,11 +1201,13 @@ module Cnvrg
     end
 
     desc '', '', :hide => true
-    def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, force: false, override: false, threads: 15, message: nil)
+    def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, force: false, override: false, threads: 15, message: nil, auto_cache: false, external_disk: nil)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
-
+        if auto_cache && external_disk.blank?
+          raise SignalException.new(1, "for auto caching external disk is required")
+        end
         owner, slug = get_owner_slug(dataset_url)
         @dataset = Dataset.new(dataset_info: {:owner =>  owner, :slug => slug})
         @datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
@@ -1228,7 +1230,7 @@ module Cnvrg
           Cnvrg::Logger.info("Put files in latest commit")
           response = @datafiles.last_valid_commit()
           unless response #means we failed in the start commit.
-            raise SignalException.new(1, "Cant put files into commit:#{commit}, check the dataset id and commitc")
+            raise SignalException.new(1, "Cant put files into commit:#{commit}, check the dataset id and commit")
           end
           @commit = response['result']['sha1']
         else
@@ -1254,7 +1256,7 @@ module Cnvrg
           raise SignalException.new(1, res.msg)
         end
         Cnvrg::Logger.info("Saving commit on server")
-        res = @datafiles.end_commit(@commit,force, success: true, commit_type: "put")
+        res = @datafiles.end_commit(@commit,force, success: true, commit_type: "put", auto_cache: auto_cache, external_disk: external_disk)
         msg = res['result']
         response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
         unless response.is_success?
@@ -1262,19 +1264,25 @@ module Cnvrg
         end
 
         log_message("Uploading files finished Successfully", Thor::Shell::Color::GREEN)
+        if msg['cache_error'].present?
+          log_message("Couldn't cache commit: #{msg['cache_error']}", Thor::Shell::Color::YELLOW)
+        end
       rescue SignalException => e
         log_message(e.message, Thor::Shell::Color::RED)
         return false
       end
     end
 
-
     desc '', '', :hide => true
-    def data_rm(dataset_url, regex_list: [], commit: '', message: nil)
+    def data_rm(dataset_url, regex_list: [], commit: '', message: nil, auto_cache: false, external_disk: nil)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
 
+        if auto_cache && external_disk.blank?
+          raise SignalException.new(1, "for auto caching external disk is required")
+        end
+
         owner, slug = get_owner_slug(dataset_url)
         @dataset = Dataset.new(dataset_info: {:owner =>  owner, :slug => slug})
         @datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
@@ -1310,7 +1318,7 @@ module Cnvrg
           offset += chunk_size
         end
 
-        res = @datafiles.end_commit(@commit,false, success: true)
+        res = @datafiles.end_commit(@commit,false, success: true, auto_cache: auto_cache, external_disk: external_disk)
         msg = res['result']
         response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
         unless response.is_success?
@@ -1318,6 +1326,9 @@ module Cnvrg
         end
 
         log_message("Deleting files finished Successfully", Thor::Shell::Color::GREEN)
+        if msg['cache_error'].present?
+          log_message("Couldn't cache commit: #{msg['cache_error']}", Thor::Shell::Color::YELLOW)
+        end
       rescue SignalException => e
         log_message(e.message, Thor::Shell::Color::RED)
         return false

data/lib/cnvrg/data.rb

@@ -81,7 +81,6 @@ module Cnvrg
       end
     end
 
-
     desc "data upload", "Upload files from local dataset directory to remote server"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
@@ -155,6 +154,7 @@ module Cnvrg
     method_option :flatten, :type => :boolean, :aliases => ["-f", "--flatten"], :default => false
     method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
     method_option :threads, :type => :numeric, :aliases => ["--threads"], :default => 15
+    method_option :cache_link, :type => :boolean, :aliases => ["--cache_link"], :default => false, :hide => true
     def clone(dataset_url)
       cli = Cnvrg::CLI.new()
       only_tree =options[:only_tree]
@@ -165,6 +165,7 @@ module Cnvrg
       soft = options[:soft]
       flatten = options[:flatten]
       threads = options[:threads]
+      cache_link = options[:cache_link]
       cli.clone_data(
           dataset_url,
           only_tree=only_tree,
@@ -175,7 +176,8 @@ module Cnvrg
           flatten: flatten,
           relative: options[:relative],
           soft: soft,
-          threads: threads
+          threads: threads,
+          cache_link: cache_link
       )
     end
 
@@ -220,6 +222,8 @@ module Cnvrg
     method_option :threads, :type => :numeric, :aliases => ["-t","--threads"], :default => 15
     method_option :chunk_size, :type => :numeric, :aliases => ["-cs","--chunk"], :default => 1000
     method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
+    method_option :auto_cache, :type => :boolean, :aliases => ["--auto_cache"], :desc => "auto_cache", :default => false
+    method_option :external_disk, :type => :string, :aliases => ["--external_disk"], :desc => "external_disk_title", :default => nil
 
     def put(dataset_url, *files)
       cli = Cnvrg::CLI.new()
@@ -231,6 +235,8 @@ module Cnvrg
       message = options[:message]
       threads = options[:threads]
       chunk_size = options[:chunk_size]
+      auto_cache = options[:auto_cache]
+      external_disk = options[:external_disk]
       cli.data_put(
         dataset_url,
         files: files,
@@ -240,16 +246,28 @@ module Cnvrg
         override: override,
         threads: threads,
         chunk_size: chunk_size,
-        message: message
+        message: message,
+        auto_cache: auto_cache,
+        external_disk: external_disk
       )
     end
 
     desc 'data rm DATASET_URL FILES_PREFIX', 'Delete selected files from remote server'
     method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
+    method_option :auto_cache, :type => :boolean, :aliases => ["--auto_cache"], :desc => "auto_cache", :default => false
+    method_option :external_disk, :type => :string, :aliases => ["--external_disk"], :desc => "external_disk_title", :default => nil
     def rm(dataset_url, *regex_list)
       cli = Cnvrg::CLI.new()
       message = options[:message]
-      cli.data_rm(dataset_url, regex_list: regex_list, message: message)
+      auto_cache = options[:auto_cache]
+      external_disk = options[:external_disk]
+      cli.data_rm(
+        dataset_url,
+        regex_list: regex_list,
+        message: message,
+        auto_cache: auto_cache,
+        external_disk: external_disk
+      )
     end
 
     desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'

data/lib/cnvrg/datafiles.rb

@@ -1243,7 +1243,7 @@ module Cnvrg
       false
     end
 
-    def end_commit(commit_sha1, force, success: true, uploaded_files: 0, commit_type: nil)
+    def end_commit(commit_sha1, force, success: true, uploaded_files: 0, commit_type: nil, auto_cache: false, external_disk: nil)
       counter = 0
       begin
         counter += 1
@@ -1255,7 +1255,9 @@ module Cnvrg
                 force:force,
                 success: success,
                 uploaded_files: uploaded_files,
-                commit_type: commit_type
+                commit_type: commit_type,
+                auto_cache: auto_cache,
+                external_disk: external_disk
             }
         )
         is_success = Cnvrg::CLI.is_response_success(response, false)
@@ -1289,8 +1291,8 @@ module Cnvrg
       response['result']['files']
     end
 
-    def get_clone_chunk(latest_id: nil, chunk_size: 1000, commit: 'latest')
-      response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST',{commit: commit, chunk_size: chunk_size, latest_id: latest_id})
+    def get_clone_chunk(latest_id: nil, chunk_size: 1000, commit: 'latest', cache_link: false)
+      response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST',{commit: commit, chunk_size: chunk_size, latest_id: latest_id, cache_link: cache_link})
       unless Cnvrg::CLI.is_response_success(response, false)
         Cnvrg::Logger.log_info("#{{commit: commit, chunk_size: chunk_size, latest_id: latest_id}}")
         return nil
@@ -1357,7 +1359,7 @@ module Cnvrg
       end
     end
 
-    def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false, flatten: false, threads: 15)
+    def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false, flatten: false, threads: 15, cache_link: false)
       begin
         refresh_storage_token
         parallel_options = {
@@ -1378,10 +1380,18 @@ module Cnvrg
               # blob
             local_path = "#{local_path}.conflict" if conflict
             storage_path = f["path"]
-            # if File.exists? local_path
-            #   Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
-            #   next
-            # end
+              # if File.exists? local_path
+              #   Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
+              #   next
+              # end
+            if cache_link
+              cached_commits = f['cached_commits']
+
+              if cached_commits.present?
+                next if @downloader.link_file(cached_commits, local_path, @dataset.title, f['name'])
+              end
+            end
+
             resp = @downloader.safe_download(storage_path, local_path)
             Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
           rescue => e

data/lib/cnvrg/downloader/client.rb

@@ -37,6 +37,21 @@ module Cnvrg
         ### need to be implemented..
       end
 
+      def link_file(cached_commits, local_path, dataset_title, file_name)
+        prepare_download(local_path)
+        cached_commits.each do |cached_commit|
+          nfs_path = "/nfs-disk/#{cached_commit}/#{dataset_title}/#{file_name}"
+          if File.exist? nfs_path
+            FileUtils.ln(nfs_path, local_path)
+            return true
+          end
+        end
+        false
+      rescue => e
+        Cnvrg::Logger.log_error(e)
+        false
+      end
+
       def safe_download(storage_path, local_path, decrypt: true)
         safe_operation(local_path) { self.download(storage_path, local_path, decrypt: decrypt) }
       end

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.11.24'
+  VERSION = '1.11.25'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.11.24
+  version: 1.11.25
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-25 00:00:00.000000000 Z
+date: 2021-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -453,7 +453,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.9
 signing_key:
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.