cnvrg 1.11.24 → 1.11.25

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e8a5d09f38ee4ce25caecdaf73b77e37fadff3b745454030cb15863210c1b319
4
- data.tar.gz: 1f3e27bc4fa778ff048e7257f2ce753009ceaf0801c4ca3f4facfdc3011ad940
3
+ metadata.gz: 4fa2d41b1f7a80c14522b607d157762fccbee612e2b5663e41f102dde8ea8b50
4
+ data.tar.gz: 1d8acf0aaeb2e47c9d9bb424a1a7f8cba2e75cddb0dced119309aa49702f21d8
5
5
  SHA512:
6
- metadata.gz: 14d79f14723eff1d8df81907183fabbe97270fd49610d4aa456fa820e4c19994c3d2bd1b7c64160f4d66a0636ff385a32b86f6fed2bedf9c0ed98c7968267c23
7
- data.tar.gz: 80b4644fc6f560e54280ab726983f4e5d89978255991b29c510452b5514019025a3dcae45e4ad5f9a36e97823909eb6d1f2ee892cc63ef5090fa78ecacf595fe
6
+ metadata.gz: 86fa4e56ddb38cce2b6e18efdeab1316570a42bfe3f450c1eb497dba99a2444a10c2042c51ff3d5da3507c1ce22a2e7cedadd203e3514b93dd0c9c494dcab785
7
+ data.tar.gz: c3388652c80427fd5e8e3e89289c853f238a2a01eddd46a04263e9e6df44feb64f2c315307ccad055ba7dad6dc8dad0c6619183fb7adcc0cf1a67e36553717d5
data/lib/cnvrg/cli.rb CHANGED
@@ -858,7 +858,7 @@ module Cnvrg
858
858
  method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
859
859
  method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
860
860
  method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
861
- def clone_data(dataset_url, only_tree=false, commit=nil, query=nil, read=false, remote=false, flatten: false, relative: false, soft: false, threads: 15)
861
+ def clone_data(dataset_url, only_tree=false, commit=nil, query=nil, read=false, remote=false, flatten: false, relative: false, soft: false, threads: 15, cache_link: false)
862
862
  begin
863
863
  verify_logged_in(false)
864
864
  log_start(__method__, args, options)
@@ -904,7 +904,7 @@ module Cnvrg
904
904
 
905
905
  commit = response["result"]["commit"]
906
906
  files_count = response["result"]["file_count"]
907
- files = @files.get_clone_chunk(commit: commit)
907
+ files = @files.get_clone_chunk(commit: commit, cache_link: cache_link)
908
908
  downloaded_files = 0
909
909
  progressbar = ProgressBar.create(:title => "Download Progress",
910
910
  :progress_mark => '=',
@@ -917,7 +917,7 @@ module Cnvrg
917
917
 
918
918
  while files['keys'].length > 0
919
919
  Cnvrg::Logger.log_info("download multiple files, #{downloaded_files.size} files downloaded")
920
- @files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read, flatten: flatten, threads: threads)
920
+ @files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read, flatten: flatten, threads: threads, cache_link: cache_link)
921
921
 
922
922
  downloaded_files += files['keys'].length
923
923
  files = @files.get_clone_chunk(commit: commit, latest_id: files['latest'])
@@ -1201,11 +1201,13 @@ module Cnvrg
1201
1201
  end
1202
1202
 
1203
1203
  desc '', '', :hide => true
1204
- def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, force: false, override: false, threads: 15, message: nil)
1204
+ def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, force: false, override: false, threads: 15, message: nil, auto_cache: false, external_disk: nil)
1205
1205
  begin
1206
1206
  verify_logged_in(false)
1207
1207
  log_start(__method__, args, options)
1208
-
1208
+ if auto_cache && external_disk.blank?
1209
+ raise SignalException.new(1, "for auto caching external disk is required")
1210
+ end
1209
1211
  owner, slug = get_owner_slug(dataset_url)
1210
1212
  @dataset = Dataset.new(dataset_info: {:owner => owner, :slug => slug})
1211
1213
  @datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
@@ -1228,7 +1230,7 @@ module Cnvrg
1228
1230
  Cnvrg::Logger.info("Put files in latest commit")
1229
1231
  response = @datafiles.last_valid_commit()
1230
1232
  unless response #means we failed in the start commit.
1231
- raise SignalException.new(1, "Cant put files into commit:#{commit}, check the dataset id and commitc")
1233
+ raise SignalException.new(1, "Cant put files into commit:#{commit}, check the dataset id and commit")
1232
1234
  end
1233
1235
  @commit = response['result']['sha1']
1234
1236
  else
@@ -1254,7 +1256,7 @@ module Cnvrg
1254
1256
  raise SignalException.new(1, res.msg)
1255
1257
  end
1256
1258
  Cnvrg::Logger.info("Saving commit on server")
1257
- res = @datafiles.end_commit(@commit,force, success: true, commit_type: "put")
1259
+ res = @datafiles.end_commit(@commit,force, success: true, commit_type: "put", auto_cache: auto_cache, external_disk: external_disk)
1258
1260
  msg = res['result']
1259
1261
  response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
1260
1262
  unless response.is_success?
@@ -1262,19 +1264,25 @@ module Cnvrg
1262
1264
  end
1263
1265
 
1264
1266
  log_message("Uploading files finished Successfully", Thor::Shell::Color::GREEN)
1267
+ if msg['cache_error'].present?
1268
+ log_message("Couldn't cache commit: #{msg['cache_error']}", Thor::Shell::Color::YELLOW)
1269
+ end
1265
1270
  rescue SignalException => e
1266
1271
  log_message(e.message, Thor::Shell::Color::RED)
1267
1272
  return false
1268
1273
  end
1269
1274
  end
1270
1275
 
1271
-
1272
1276
  desc '', '', :hide => true
1273
- def data_rm(dataset_url, regex_list: [], commit: '', message: nil)
1277
+ def data_rm(dataset_url, regex_list: [], commit: '', message: nil, auto_cache: false, external_disk: nil)
1274
1278
  begin
1275
1279
  verify_logged_in(false)
1276
1280
  log_start(__method__, args, options)
1277
1281
 
1282
+ if auto_cache && external_disk.blank?
1283
+ raise SignalException.new(1, "for auto caching external disk is required")
1284
+ end
1285
+
1278
1286
  owner, slug = get_owner_slug(dataset_url)
1279
1287
  @dataset = Dataset.new(dataset_info: {:owner => owner, :slug => slug})
1280
1288
  @datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
@@ -1310,7 +1318,7 @@ module Cnvrg
1310
1318
  offset += chunk_size
1311
1319
  end
1312
1320
 
1313
- res = @datafiles.end_commit(@commit,false, success: true)
1321
+ res = @datafiles.end_commit(@commit,false, success: true, auto_cache: auto_cache, external_disk: external_disk)
1314
1322
  msg = res['result']
1315
1323
  response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
1316
1324
  unless response.is_success?
@@ -1318,6 +1326,9 @@ module Cnvrg
1318
1326
  end
1319
1327
 
1320
1328
  log_message("Deleting files finished Successfully", Thor::Shell::Color::GREEN)
1329
+ if msg['cache_error'].present?
1330
+ log_message("Couldn't cache commit: #{msg['cache_error']}", Thor::Shell::Color::YELLOW)
1331
+ end
1321
1332
  rescue SignalException => e
1322
1333
  log_message(e.message, Thor::Shell::Color::RED)
1323
1334
  return false
data/lib/cnvrg/data.rb CHANGED
@@ -81,7 +81,6 @@ module Cnvrg
81
81
  end
82
82
  end
83
83
 
84
-
85
84
  desc "data upload", "Upload files from local dataset directory to remote server"
86
85
  method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
87
86
  method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
@@ -155,6 +154,7 @@ module Cnvrg
155
154
  method_option :flatten, :type => :boolean, :aliases => ["-f", "--flatten"], :default => false
156
155
  method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
157
156
  method_option :threads, :type => :numeric, :aliases => ["--threads"], :default => 15
157
+ method_option :cache_link, :type => :boolean, :aliases => ["--cache_link"], :default => false, :hide => true
158
158
  def clone(dataset_url)
159
159
  cli = Cnvrg::CLI.new()
160
160
  only_tree =options[:only_tree]
@@ -165,6 +165,7 @@ module Cnvrg
165
165
  soft = options[:soft]
166
166
  flatten = options[:flatten]
167
167
  threads = options[:threads]
168
+ cache_link = options[:cache_link]
168
169
  cli.clone_data(
169
170
  dataset_url,
170
171
  only_tree=only_tree,
@@ -175,7 +176,8 @@ module Cnvrg
175
176
  flatten: flatten,
176
177
  relative: options[:relative],
177
178
  soft: soft,
178
- threads: threads
179
+ threads: threads,
180
+ cache_link: cache_link
179
181
  )
180
182
  end
181
183
 
@@ -220,6 +222,8 @@ module Cnvrg
220
222
  method_option :threads, :type => :numeric, :aliases => ["-t","--threads"], :default => 15
221
223
  method_option :chunk_size, :type => :numeric, :aliases => ["-cs","--chunk"], :default => 1000
222
224
  method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
225
+ method_option :auto_cache, :type => :boolean, :aliases => ["--auto_cache"], :desc => "auto_cache", :default => false
226
+ method_option :external_disk, :type => :string, :aliases => ["--external_disk"], :desc => "external_disk_title", :default => nil
223
227
 
224
228
  def put(dataset_url, *files)
225
229
  cli = Cnvrg::CLI.new()
@@ -231,6 +235,8 @@ module Cnvrg
231
235
  message = options[:message]
232
236
  threads = options[:threads]
233
237
  chunk_size = options[:chunk_size]
238
+ auto_cache = options[:auto_cache]
239
+ external_disk = options[:external_disk]
234
240
  cli.data_put(
235
241
  dataset_url,
236
242
  files: files,
@@ -240,16 +246,28 @@ module Cnvrg
240
246
  override: override,
241
247
  threads: threads,
242
248
  chunk_size: chunk_size,
243
- message: message
249
+ message: message,
250
+ auto_cache: auto_cache,
251
+ external_disk: external_disk
244
252
  )
245
253
  end
246
254
 
247
255
  desc 'data rm DATASET_URL FILES_PREFIX', 'Delete selected files from remote server'
248
256
  method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
257
+ method_option :auto_cache, :type => :boolean, :aliases => ["--auto_cache"], :desc => "auto_cache", :default => false
258
+ method_option :external_disk, :type => :string, :aliases => ["--external_disk"], :desc => "external_disk_title", :default => nil
249
259
  def rm(dataset_url, *regex_list)
250
260
  cli = Cnvrg::CLI.new()
251
261
  message = options[:message]
252
- cli.data_rm(dataset_url, regex_list: regex_list, message: message)
262
+ auto_cache = options[:auto_cache]
263
+ external_disk = options[:external_disk]
264
+ cli.data_rm(
265
+ dataset_url,
266
+ regex_list: regex_list,
267
+ message: message,
268
+ auto_cache: auto_cache,
269
+ external_disk: external_disk
270
+ )
253
271
  end
254
272
 
255
273
  desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'
@@ -1243,7 +1243,7 @@ module Cnvrg
1243
1243
  false
1244
1244
  end
1245
1245
 
1246
- def end_commit(commit_sha1, force, success: true, uploaded_files: 0, commit_type: nil)
1246
+ def end_commit(commit_sha1, force, success: true, uploaded_files: 0, commit_type: nil, auto_cache: false, external_disk: nil)
1247
1247
  counter = 0
1248
1248
  begin
1249
1249
  counter += 1
@@ -1255,7 +1255,9 @@ module Cnvrg
1255
1255
  force:force,
1256
1256
  success: success,
1257
1257
  uploaded_files: uploaded_files,
1258
- commit_type: commit_type
1258
+ commit_type: commit_type,
1259
+ auto_cache: auto_cache,
1260
+ external_disk: external_disk
1259
1261
  }
1260
1262
  )
1261
1263
  is_success = Cnvrg::CLI.is_response_success(response, false)
@@ -1289,8 +1291,8 @@ module Cnvrg
1289
1291
  response['result']['files']
1290
1292
  end
1291
1293
 
1292
- def get_clone_chunk(latest_id: nil, chunk_size: 1000, commit: 'latest')
1293
- response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST',{commit: commit, chunk_size: chunk_size, latest_id: latest_id})
1294
+ def get_clone_chunk(latest_id: nil, chunk_size: 1000, commit: 'latest', cache_link: false)
1295
+ response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST',{commit: commit, chunk_size: chunk_size, latest_id: latest_id, cache_link: cache_link})
1294
1296
  unless Cnvrg::CLI.is_response_success(response, false)
1295
1297
  Cnvrg::Logger.log_info("#{{commit: commit, chunk_size: chunk_size, latest_id: latest_id}}")
1296
1298
  return nil
@@ -1357,7 +1359,7 @@ module Cnvrg
1357
1359
  end
1358
1360
  end
1359
1361
 
1360
- def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false, flatten: false, threads: 15)
1362
+ def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false, flatten: false, threads: 15, cache_link: false)
1361
1363
  begin
1362
1364
  refresh_storage_token
1363
1365
  parallel_options = {
@@ -1378,10 +1380,18 @@ module Cnvrg
1378
1380
  # blob
1379
1381
  local_path = "#{local_path}.conflict" if conflict
1380
1382
  storage_path = f["path"]
1381
- # if File.exists? local_path
1382
- # Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
1383
- # next
1384
- # end
1383
+ # if File.exists? local_path
1384
+ # Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
1385
+ # next
1386
+ # end
1387
+ if cache_link
1388
+ cached_commits = f['cached_commits']
1389
+
1390
+ if cached_commits.present?
1391
+ next if @downloader.link_file(cached_commits, local_path, @dataset.title, f['name'])
1392
+ end
1393
+ end
1394
+
1385
1395
  resp = @downloader.safe_download(storage_path, local_path)
1386
1396
  Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
1387
1397
  rescue => e
@@ -37,6 +37,21 @@ module Cnvrg
37
37
  ### need to be implemented..
38
38
  end
39
39
 
40
+ def link_file(cached_commits, local_path, dataset_title, file_name)
41
+ prepare_download(local_path)
42
+ cached_commits.each do |cached_commit|
43
+ nfs_path = "/nfs-disk/#{cached_commit}/#{dataset_title}/#{file_name}"
44
+ if File.exist? nfs_path
45
+ FileUtils.ln(nfs_path, local_path)
46
+ return true
47
+ end
48
+ end
49
+ false
50
+ rescue => e
51
+ Cnvrg::Logger.log_error(e)
52
+ false
53
+ end
54
+
40
55
  def safe_download(storage_path, local_path, decrypt: true)
41
56
  safe_operation(local_path) { self.download(storage_path, local_path, decrypt: decrypt) }
42
57
  end
data/lib/cnvrg/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Cnvrg
2
- VERSION = '1.11.24'
2
+ VERSION = '1.11.25'
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cnvrg
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.11.24
4
+ version: 1.11.25
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2021-02-25 00:00:00.000000000 Z
13
+ date: 2021-02-27 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: bundler
@@ -453,7 +453,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
453
453
  - !ruby/object:Gem::Version
454
454
  version: '0'
455
455
  requirements: []
456
- rubygems_version: 3.1.2
456
+ rubygems_version: 3.0.9
457
457
  signing_key:
458
458
  specification_version: 4
459
459
  summary: A CLI tool for interacting with cnvrg.io.