cnvrg 1.11.15 → 1.11.24

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8798ab84db1e0742dc733a0f74b5830e2c5396099fc52bdb4a04ef108a43408e
4
- data.tar.gz: 795988ec46ba506091136240bda258f923469e4b3000d58e84371df09f65840d
3
+ metadata.gz: e8a5d09f38ee4ce25caecdaf73b77e37fadff3b745454030cb15863210c1b319
4
+ data.tar.gz: 1f3e27bc4fa778ff048e7257f2ce753009ceaf0801c4ca3f4facfdc3011ad940
5
5
  SHA512:
6
- metadata.gz: '0497d165396b52d2975c3d3ab8c097596c173f04c63711519cf2c122fb9dc5f90ecc493aab7ba2cba0d7064dc14b52a7e1d7f2f6f5e68d3af020d4b3f1f5b35c'
7
- data.tar.gz: 72c65364f4c9ad8d5acf68546f0a428c113dd9697613674ce83b52a3ce7fcec8064a7517d9b5b5ae7f21a8690ba935d028622688d72f87eb3e1927d7fe5ea9e2
6
+ metadata.gz: 14d79f14723eff1d8df81907183fabbe97270fd49610d4aa456fa820e4c19994c3d2bd1b7c64160f4d66a0636ff385a32b86f6fed2bedf9c0ed98c7968267c23
7
+ data.tar.gz: 80b4644fc6f560e54280ab726983f4e5d89978255991b29c510452b5514019025a3dcae45e4ad5f9a36e97823909eb6d1f2ee892cc63ef5090fa78ecacf595fe
data/cnvrg.gemspec CHANGED
@@ -23,8 +23,8 @@ Gem::Specification.new do |spec|
23
23
  spec.add_development_dependency 'rspec', '~> 3.0'
24
24
  spec.add_development_dependency 'vcr', '~> 3.0'
25
25
  spec.add_development_dependency 'aruba'
26
- spec.add_development_dependency 'pry'
27
-
26
+ spec.add_development_dependency 'pry'
27
+
28
28
  spec.add_runtime_dependency 'mimemagic', '~> 0.3.1','>=0.3.2'
29
29
  spec.add_runtime_dependency 'faraday', '~> 0.15.2'
30
30
  spec.add_runtime_dependency 'netrc', '~> 0.11.0'
data/lib/cnvrg/api.rb CHANGED
@@ -31,6 +31,10 @@ module Cnvrg
31
31
  end
32
32
  def self.request(resource, method = 'GET', data = {}, parse_request = true)
33
33
  resource = URI::encode resource
34
+
35
+ # We need to remoe all double slashes from the url to work with the proxy
36
+ resource = resource.gsub(/[\/]{2,}/, "/").gsub("https:/", "https://").gsub("http:/", "http://")
37
+
34
38
  begin
35
39
  n = Netrc.read
36
40
  rescue => e
data/lib/cnvrg/cli.rb CHANGED
@@ -1210,6 +1210,7 @@ module Cnvrg
1210
1210
  @dataset = Dataset.new(dataset_info: {:owner => owner, :slug => slug})
1211
1211
  @datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
1212
1212
  @files = @datafiles.verify_files_exists(files)
1213
+ @files = @files.uniq { |t| t.gsub('./', '')}
1213
1214
 
1214
1215
  if @files.blank?
1215
1216
  raise SignalException.new(1, "Cant find files to upload, exiting.")
@@ -2346,7 +2347,6 @@ module Cnvrg
2346
2347
  log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
2347
2348
  return true
2348
2349
  end
2349
- force = true
2350
2350
  end
2351
2351
 
2352
2352
  if ignore.nil? or ignore.empty?
@@ -3323,7 +3323,7 @@ module Cnvrg
3323
3323
  real: real_time
3324
3324
  }
3325
3325
  if print_log
3326
- puts line
3326
+ puts({log: line, timestamp: Time.now, exp_logs: true}.to_json)
3327
3327
  end
3328
3328
  log << cur_log
3329
3329
  if log.size >= 10
@@ -43,8 +43,11 @@ module Cnvrg
43
43
  def verify_files_exists(files)
44
44
  paths = []
45
45
  files.each do |file|
46
+ # dir shouldnt have ending slash.
47
+ file = file[0..-2] if file.end_with? '/'
46
48
  if File.exists? file
47
49
  if File.directory? file
50
+ paths << file unless file == '.'
48
51
  paths += Dir.glob("#{file}/**/*")
49
52
  else
50
53
  paths << file
@@ -349,11 +352,11 @@ module Cnvrg
349
352
  progress_mutex = Mutex.new
350
353
  file_queue = Queue.new
351
354
  progress_queue = Queue.new
355
+ dirs_queue = Queue.new
352
356
  worker_threads = []
353
357
  progress_threads = []
354
358
 
355
359
  # Vars to keep track of uploaded files and directories
356
- dirs = []
357
360
  uploaded_files = []
358
361
 
359
362
  begin
@@ -376,6 +379,29 @@ module Cnvrg
376
379
  end
377
380
  end
378
381
 
382
+ dir_thread = Thread.new do
383
+ dirs_to_create = []
384
+ loop do
385
+ progress_mutex.synchronize {
386
+ dir = dirs_queue.deq(non_block: true) rescue nil
387
+ dirs_to_create << dir unless dir.nil?
388
+ }
389
+ if dirs_to_create.size >= 1000 || progressbar.finished?
390
+ resp = Cnvrg::API.request(@base_resource + "create_dirs", "POST", {dirs: dirs_to_create, commit_sha1: commit_sha1})
391
+
392
+ break if resp == false # if resp is false it means 404 which is old server
393
+ unless Cnvrg::CLI.is_response_success(resp, false)
394
+ time = Time.current
395
+ Cnvrg::Logger.log_error_message("Failed to create dirs: #{time}, #{resp.try(:fetch, "message")}")
396
+ dirs_to_create = []
397
+ next
398
+ end
399
+ dirs_to_create = []
400
+ end
401
+ break if progressbar.finished? && dirs_queue.empty? && dirs_to_create.empty?
402
+ end
403
+ end
404
+
379
405
  # init the thread that handles the file upload progress and saving them in the server
380
406
  threads.times do |i|
381
407
  progress_threads[i] = Thread.new do
@@ -383,7 +409,6 @@ module Cnvrg
383
409
  file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
384
410
  unless file.nil?
385
411
  blob_ids = []
386
- dirs_to_upload = []
387
412
 
388
413
  progress_mutex.synchronize {
389
414
  progressbar.progress += 1
@@ -391,9 +416,7 @@ module Cnvrg
391
416
 
392
417
  if uploaded_files.size >= chunk_size or progressbar.finished?
393
418
  blob_ids = uploaded_files.map {|f| f['bv_id']}
394
- dirs_to_upload = dirs.clone
395
419
  uploaded_files = []
396
- dirs = []
397
420
  end
398
421
  }
399
422
 
@@ -401,10 +424,9 @@ module Cnvrg
401
424
  refresh_storage_token
402
425
  Cnvrg::Logger.info("Finished upload chunk of #{chunk_size} files, Sending Upload files save")
403
426
 
404
-
405
427
  retry_count = 0
406
428
  loop do
407
- upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs_to_upload})
429
+ upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids})
408
430
 
409
431
  if not (Cnvrg::CLI.is_response_success(upload_resp, false))
410
432
  retry_count += 1
@@ -427,8 +449,8 @@ module Cnvrg
427
449
 
428
450
  if progressbar.finished?
429
451
  Cnvrg::Logger.info("Progress bar finished closing queues")
430
- file_queue.close()
431
- progress_queue.close()
452
+ file_queue.close
453
+ progress_queue.close
432
454
  Thread.exit
433
455
  end
434
456
  end
@@ -441,7 +463,21 @@ module Cnvrg
441
463
  files_chunk = chunk.map{|p| p.gsub(/^\.\//, '')}
442
464
  Cnvrg::Logger.info("Generating chunk idx")
443
465
  tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads, cli: cli)
466
+
467
+ progress_mutex.synchronize {
468
+ # Handle directories:
469
+ new_dirs = tree.keys.select { |k| tree[k].nil? }
470
+
471
+ if new_dirs.blank?
472
+ ## we need to send 1 file so we will inflated dirs from in case when we dont have folders in the tree
473
+ file = tree.keys.find { |k| tree[k] != nil }
474
+ dirs_queue.push file
475
+ end
476
+
477
+ new_dirs.each { |dir| dirs_queue.push dir }
478
+ }
444
479
  Cnvrg::Logger.info("Getting files info from server")
480
+
445
481
  results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
446
482
  next unless results
447
483
 
@@ -450,11 +486,8 @@ module Cnvrg
450
486
  next
451
487
  end
452
488
 
453
- # Handle directories:
454
- new_dirs = tree.keys.select {|k| tree[k].nil?}
455
- dirs += new_dirs
456
-
457
489
  files_to_upload = results['files']
490
+
458
491
  progress_mutex.synchronize {
459
492
  progressbar.progress += tree.keys.length - files_to_upload.length
460
493
  }
@@ -466,7 +499,10 @@ module Cnvrg
466
499
  file_queue.push tree[key].merge(files_to_upload[key])
467
500
  end
468
501
  end
502
+
469
503
  Cnvrg::Logger.info("Waiting to progress and workers to finish")
504
+ dir_thread.join
505
+ dirs_queue.close
470
506
  progress_threads.each(&:join)
471
507
  worker_threads.each(&:join)
472
508
  Thread.report_on_exception = true
data/lib/cnvrg/dataset.rb CHANGED
@@ -564,7 +564,8 @@ module Cnvrg
564
564
  safe_path = file
565
565
  safe_path = file[1..-1] if file.start_with? "/"
566
566
 
567
- label = safe_path.gsub(self.local_path + "/", "")
567
+ dataset_local_path = self.local_path + "/"
568
+ label = safe_path.start_with?(dataset_local_path) ? safe_path.sub(dataset_local_path, "") : safe_path
568
569
  label = "#{prefix}/#{label}" if prefix.present?
569
570
  if not Cnvrg::Files.valid_file_name?(label)
570
571
  if cli
@@ -598,6 +599,7 @@ module Cnvrg
598
599
  }
599
600
  end
600
601
  end
602
+
601
603
  if prefix.present? #add the prefix as dirs to the files
602
604
  #lets say the prefix is a/b/c so we want that a/, a/b/, a/b/c/ will be in our files_list
603
605
  dirs = prefix.split('/')
@@ -17,7 +17,9 @@ module Cnvrg
17
17
  count = 0
18
18
  begin
19
19
  count += 1
20
- sts = open(sts_path, {ssl_verify_mode: 0}).read rescue nil
20
+ sts_file = open(sts_path, {ssl_verify_mode: 0})
21
+ sts = sts_file.read
22
+ sts.split("\n")
21
23
  rescue => e
22
24
  backoff_time_seconds = backoff_time(count)
23
25
  sleep backoff_time_seconds
@@ -25,7 +27,6 @@ module Cnvrg
25
27
  retry if count <= 20
26
28
  raise StandardError.new("Cant access storage: #{e.message}")
27
29
  end
28
- sts.split("\n")
29
30
  end
30
31
 
31
32
  def cut_prefix(prefix, file)
data/lib/cnvrg/files.rb CHANGED
@@ -787,6 +787,7 @@ module Cnvrg
787
787
  end
788
788
  else
789
789
  file_path += postfix
790
+
790
791
  # blob
791
792
  begin
792
793
  if not File.exists?(project_home+"/"+File.dirname(file_path))
@@ -795,12 +796,19 @@ module Cnvrg
795
796
  local_path = project_home+"/"+file_path
796
797
  storage_path = f["path"]
797
798
  @client.safe_download(storage_path, local_path)
798
- progress.progress += 1 if progress.present?
799
- download_succ_count += 1
800
799
  rescue => e
801
800
  cli.log_message("Could not download file #{file_path}. error: #{e.message}", Thor::Shell::Color::RED)
802
801
  raise e
803
802
  end
803
+
804
+ # progressbar can throw an exception so we no longer trust it!
805
+ begin
806
+ progress.progress += 1 if progress.present?
807
+ rescue
808
+ nil
809
+ ensure
810
+ download_succ_count += 1
811
+ end
804
812
  end
805
813
  end
806
814
  if download_succ_count == files["keys"].size
data/lib/cnvrg/flow.rb CHANGED
@@ -62,7 +62,7 @@ module Cnvrg
62
62
  resp = Cnvrg::API.request(url, 'POST', {flow_version: recipe.to_json}) || {}
63
63
  if resp["status"] == 200
64
64
  return [Flows.new(resp["flow_version"]["flow_id"], project: project), resp["flow_version"]["id"]]
65
- elsif resp["status"] == 400
65
+ elsif resp["status"].between?(400,499)
66
66
  raise StandardError.new(resp["message"])
67
67
  end
68
68
  raise StandardError.new("Can't create new flow")
data/lib/cnvrg/project.rb CHANGED
@@ -448,8 +448,8 @@ module Cnvrg
448
448
  next
449
449
  end
450
450
  if File.directory? e
451
-
452
- tree_idx[label + "/"] = nil
451
+ dir_name = (label.ends_with? "/") ? label : (label + "/")
452
+ tree_idx[dir_name] = nil
453
453
  else
454
454
  file_in_idx = old_idx[:tree][label] rescue nil
455
455
  last_modified = File.mtime(e).to_f
@@ -513,10 +513,11 @@ module Cnvrg
513
513
  #upload
514
514
  local_idx = self.generate_idx(deploy: deploy, files: specific_files)
515
515
  end
516
+
516
517
  commit = local_idx[:commit]
517
518
  tree = local_idx[:tree]
518
519
  ignore_list = self.send_ignore_list()
519
- if force or specific_files.present?
520
+ if force
520
521
  added = []
521
522
  if tree.present?
522
523
  added += local_idx[:tree].keys
data/lib/cnvrg/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Cnvrg
2
- VERSION = '1.11.15'
3
- end
2
+ VERSION = '1.11.24'
3
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cnvrg
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.11.15
4
+ version: 1.11.24
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2021-01-25 00:00:00.000000000 Z
13
+ date: 2021-02-25 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: bundler