cnvrg 0.7.7 → 0.7.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -16,7 +16,121 @@ module Cnvrg
16
16
  @project_slug = project_slug
17
17
  @owner = owner
18
18
  @base_resource = "users/#{owner}/projects/#{project_slug}/"
19
- @project_home = project_home
19
+ @project_home = project_home.presence || Cnvrg::CLI.get_project_home
20
+ end
21
+
22
+ def get_upload_options(number_of_items: 0, progress: false)
23
+ options = {
24
+ in_processes: Cnvrg::CLI::ParallelProcesses,
25
+ in_thread: Cnvrg::CLI::ParallelThreads,
26
+ isolation: true
27
+ }
28
+ if progress
29
+ options[:progress] = {
30
+ :title => "Upload Progress",
31
+ :progress_mark => '=',
32
+ :format => "%b>>%i| %p%% %t",
33
+ :starting_at => 0,
34
+ :total => number_of_items,
35
+ :autofinish => true
36
+ }
37
+ end
38
+ options
39
+ end
40
+
41
+ def upload_files_old(files_list, commit_sha1, progress: nil)
42
+ Parallel.map(files_list) do |file|
43
+ Cnvrg::Helpers.try_until_success{self.upload_file("#{@project_home}/#{file}", file, commit_sha1)}
44
+ progress.progress += 1
45
+ end
46
+ end
47
+
48
+ def upload_multiple_files(files_list, commit_sha1, progress: nil)
49
+ #open files on the server.
50
+ Cnvrg::Logger.log_info("Uploading project files")
51
+ return if files_list.blank?
52
+ if Cnvrg::Helpers.server_version < 1
53
+ return self.upload_files_old(files_list, commit_sha1, progress: progress)
54
+ end
55
+ files_list = files_list.map{|x| [x,self.parse_file(x)]}.to_h
56
+ resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST', {files: files_list, commit: commit_sha1})
57
+ unless Cnvrg::CLI.is_response_success(resp, false)
58
+ raise Exception.new("Cant upload files to the server.")
59
+ end
60
+
61
+ # resolve bucket
62
+ res = resp['result']
63
+ files = res['files']
64
+ props = Cnvrg::Helpers.get_s3_props(res)
65
+ client = props[:client]
66
+ bucket = props[:bucket]
67
+ upload_options = props[:upload_options]
68
+ s3_bucket = Aws::S3::Resource.new(client: client).bucket(bucket)
69
+ #upload files
70
+ files.keys.map do |file|
71
+ # Parallel.map(files.keys, self.get_upload_options) do |file|
72
+ resp = Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]), s3_bucket, options: upload_options)}
73
+ raise Exception.new("Cant upload #{file}") unless resp
74
+ progress.progress += 1 if progress.present?
75
+ end
76
+
77
+ #save files on the server.
78
+ blob_ids = files.values.map {|f| f['bv_id']}
79
+ resp = Cnvrg::API.request(@base_resource + "upload_files_save", 'POST', {blob_ids: blob_ids, commit: commit_sha1})
80
+ unless Cnvrg::CLI.is_response_success(resp, false)
81
+ raise Exception.new("Cant save uploaded files to the server.")
82
+ end
83
+ end
84
+
85
+
86
+ def delete_files_from_server_old(files, commit_sha1)
87
+ #files are absolute path here.
88
+ files.each do |file|
89
+ if file.ends_with? '/'
90
+ #dir
91
+ self.delete_dir(file, commit_sha1)
92
+ else
93
+ #file
94
+ self.delete_file(file, commit_sha1)
95
+ end
96
+ end
97
+ end
98
+
99
+ def delete_files_from_server(files, commit_sha1)
100
+ #files are absolute path files here. ^^
101
+ if Cnvrg::Helpers.server_version < 1
102
+ return self.delete_files_from_server_old(files, commit_sha1)
103
+ end
104
+ #convert files to relative path
105
+ files = files.map{|file| file.gsub(/^#{@project_home + "/"}/, "")}
106
+ return if files.blank?
107
+ resp = Cnvrg::API.request(@base_resource + "delete_files", 'DELETE', {files: files, commit: commit_sha1})
108
+ unless Cnvrg::CLI.is_response_success(resp, false)
109
+ raise Exception.new("Cant delete the following files from the server.")
110
+ end
111
+ end
112
+
113
+ def upload_single_file(file, bucket, options: {})
114
+ path = file['path']
115
+ absolute_path = file[:absolute_path]
116
+ resp = bucket.object(path).
117
+ upload_file(absolute_path, options)
118
+ unless resp
119
+ raise Exception.new("Cant upload #{absolute_path}")
120
+ end
121
+ resp
122
+ end
123
+
124
+ def parse_file(file)
125
+ abs_path = "#{@project_home}/#{file}"
126
+ return {relative_path: file, absolute_path: abs_path} if file.ends_with? '/'
127
+ file_name = File.basename(file)
128
+ file_size = File.size abs_path
129
+ mime_type = MimeMagic.by_path(abs_path)
130
+ content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
131
+ sha1 = OpenSSL::Digest::SHA1.file(abs_path).hexdigest
132
+
133
+ {relative_path: file, absolute_path: abs_path, file_name: file_name, file_size: file_size, content_type: content_type, sha1: sha1}
20
134
  end
21
135
 
22
136
  def upload_file(absolute_path, relative_path, commit_sha1)
@@ -30,24 +144,9 @@ module Cnvrg
30
144
  file_size: file_size, file_content_type: content_type, sha1: sha1,
31
145
  new_version:true,only_large:true})
32
146
 
33
-
34
147
  if Cnvrg::CLI.is_response_success(upload_resp, false)
35
-
36
- path = upload_resp["result"]["path"]
37
148
  s3_res = upload_large_files_s3(upload_resp, absolute_path)
38
-
39
- # if file_size.to_f>= Cnvrg::Files::LARGE_FILE.to_f
40
- # s3_res = upload_large_files_s3(upload_resp, absolute_path)
41
- # else
42
- # s3_res = upload_small_files_s3(path, absolute_path, content_type)
43
- # end
44
149
  return s3_res
45
- # if s3_res
46
- # update_s3_resp = Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
47
- # blob_id: upload_resp["result"]["id"]})
48
- # is_suc = Cnvrg::CLI.is_response_success(update_s3_resp, false)
49
- #
50
- # return is_suc
51
150
  end
52
151
  return false
53
152
 
@@ -248,14 +347,78 @@ module Cnvrg
248
347
  end
249
348
 
250
349
  rescue => e
251
- puts e
350
+ Cnvrg::Logger.log_error(e)
252
351
  return false
352
+ end
353
+
354
+ def resolve_bucket(response)
355
+ begin
356
+ sts_path = response["path_sts"]
357
+ sts_body = self.download_and_read(sts_path)
358
+ split = sts_body.split("\n")
359
+ key = split[0]
360
+ iv = split[1]
361
+ access = Cnvrg::Helpers.decrypt(key, iv, response["sts_a"])
362
+
363
+ secret = Cnvrg::Helpers.decrypt(key, iv, response["sts_s"])
364
+
365
+ session = Cnvrg::Helpers.decrypt(key, iv, response["sts_st"])
366
+ region = Cnvrg::Helpers.decrypt(key, iv, response["region"])
367
+
368
+ bucket = Cnvrg::Helpers.decrypt(key, iv, response["bucket"])
369
+ Cnvrg::Logger.log_info("Resolving bucket #{bucket}, region: #{region}")
370
+ is_s3 = response["is_s3"]
371
+ if is_s3 or is_s3.nil?
372
+ client = Aws::S3::Client.new(
373
+ :access_key_id => access,
374
+ :secret_access_key => secret,
375
+ :session_token => session,
376
+ :region => region,
377
+ :use_accelerate_endpoint => true,
378
+ :http_open_timeout => 60, :retry_limit => 20)
379
+ else
380
+ endpoint = Cnvrg::Helpers.decrypt(key, iv, response["endpoint"])
381
+ client = Aws::S3::Client.new(
382
+ :access_key_id => access,
383
+ :secret_access_key => secret,
384
+ :region => region,
385
+ :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
386
+ :use_accelerate_endpoint => false,
387
+ :server_side_encryption => 'AES256',
388
+ :http_open_timeout => 60, :retry_limit => 20)
389
+ end
253
390
 
391
+ s3 = Aws::S3::Resource.new(client: client)
392
+ s3.bucket(bucket)
393
+ rescue => e
394
+ Cnvrg::Logger.log_error(e)
395
+ Cnvrg::Logger.log_method(bind: binding)
396
+ end
397
+ end
254
398
 
399
+ def download_and_read(path)
400
+ body = nil
401
+ retries = 0
402
+ success= false
403
+ while !success and retries < 20
404
+ begin
405
+ if !Helpers.is_verify_ssl
406
+ body = open(path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
407
+ else
408
+ body = open(path).read
409
+ end
410
+ success = true
411
+ rescue => e
412
+ retries +=1
413
+ sleep(1)
414
+ end
415
+ end
416
+ body
255
417
  end
256
418
 
257
419
  def upload_large_files_s3(upload_resp, file_path)
258
420
  begin
421
+ return true if upload_resp['result']['already_exists'].present?
259
422
  sts_path = upload_resp["result"]["path_sts"]
260
423
 
261
424
  retries = 0
@@ -360,13 +523,13 @@ module Cnvrg
360
523
 
361
524
  end
362
525
 
363
- def delete_file(absolute_path, relative_path, commit_sha1)
364
- response = Cnvrg::API.request(@base_resource + "delete_file", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
526
+ def delete_file(relative_path, commit_sha1)
527
+ response = Cnvrg::API.request(@base_resource + "delete_file", 'DELETE', {relative_path: relative_path, commit_sha1: commit_sha1})
365
528
  return Cnvrg::CLI.is_response_success(response, false)
366
529
  end
367
530
 
368
- def delete_dir(absolute_path, relative_path, commit_sha1)
369
- response = Cnvrg::API.request(@base_resource + "delete_dir", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
531
+ def delete_dir(relative_path, commit_sha1)
532
+ response = Cnvrg::API.request(@base_resource + "delete_dir", 'DELETE', {relative_path: relative_path, commit_sha1: commit_sha1})
370
533
  return Cnvrg::CLI.is_response_success(response, false)
371
534
  end
372
535
 
@@ -376,6 +539,18 @@ module Cnvrg
376
539
  end
377
540
 
378
541
 
542
+ def calculate_sha1(files_list)
543
+ files_list = files_list.map{|file| "#{@project_home}/#{file}"}
544
+ files_list = files_list.select{|file| !file.ends_with? '/'}
545
+ #TODO: parallel
546
+ files_list.map do |file|
547
+ next [file, nil] unless File.exists? file
548
+ sha1 = OpenSSL::Digest::SHA1.file(file).hexdigest
549
+ [file.gsub("#{@project_home}/", ""), sha1]
550
+ end.to_h
551
+ end
552
+
553
+
379
554
  def download_file_s3(absolute_path, relative_path, project_home, commit_sha1=nil, conflict=false)
380
555
  begin
381
556
  res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path,
@@ -456,73 +631,78 @@ module Cnvrg
456
631
 
457
632
  end
458
633
  end
459
- def download_multpile_files_s3(files, project_home)
634
+
635
+ def download_files(files, commit, postfix: '', progress: nil)
636
+ res = Cnvrg::API.request(@base_resource + "download_files", 'POST', {files: files, commit: commit})
637
+ unless Cnvrg::CLI.is_response_success(res, false)
638
+ raise Exception.new("Cant download files from the server.")
639
+ end
640
+ self.download_multpile_files_s3(res['result'], @project_home, postfix: postfix, progress: progress)
641
+ end
642
+
643
+ def delete_files_local(deleted, conflicted: [], progress: nil)
644
+ deleted -= conflicted
645
+ deleted.each{|file| self.delete(file); progress.progress += 1 if progress.present?}
646
+ conflicted.each{|file| self.delete_conflict(file); progress.progress += 1 if progress.present?}
647
+ end
648
+
649
+ def download_multpile_files_s3(files, project_home, postfix: '', progress: nil)
460
650
  begin
461
- props = Cnvrg::Helpers.get_s3_props(files)
462
- client = props[:client]
463
- iv = props[:iv]
464
- key = props[:key]
465
- bucket = props[:bucket]
466
- download_succ_count = 0
467
- parallel_options = {
468
- :progress => {
469
- :title => "Download Progress",
470
- :progress_mark => '=',
471
- :format => "%b>>%i| %p%% %t",
472
- :starting_at => 0,
473
- :total => files["keys"].size,
474
- :autofinish => true
475
- },
476
- in_threads: Cnvrg::Helpers.parallel_threads,
477
- isolation: true
478
- }
479
- Parallel.map(files["keys"], parallel_options) do |f|
480
-
481
- file_path = f["name"]
482
- if file_path.end_with? "/"
483
- # dir
484
- if download_dir(file_path, file_path, project_home)
485
- download_succ_count += 1
486
- else
487
- return Cnvrg::Result.new(false,"Could not create directory: #{file_path}")
488
- raise Parallel::Kill
489
- end
651
+ props = Cnvrg::Helpers.get_s3_props(files)
652
+ client = props[:client]
653
+ iv = props[:iv]
654
+ key = props[:key]
655
+ bucket = props[:bucket]
656
+ download_succ_count = 0
657
+ parallel_options = {
658
+ in_threads: Cnvrg::Helpers.parallel_threads,
659
+ isolation: true
660
+ }
661
+ Parallel.map(files["keys"], parallel_options) do |f|
662
+
663
+ file_path = f["name"]
664
+ if file_path.end_with? "/"
665
+ # dir
666
+ if download_dir(file_path, file_path, project_home)
667
+ download_succ_count += 1
490
668
  else
491
- # blob
492
- begin
493
- if not File.exists?(project_home+"/"+File.dirname(file_path))
494
- FileUtils.makedirs(project_home+"/"+File.dirname(file_path))
495
- end
669
+ return Cnvrg::Result.new(false,"Could not create directory: #{file_path}")
670
+ raise Parallel::Kill
671
+ end
672
+ else
673
+ file_path += postfix
674
+ # blob
675
+ begin
676
+ if not File.exists?(project_home+"/"+File.dirname(file_path))
677
+ FileUtils.makedirs(project_home+"/"+File.dirname(file_path))
678
+ end
496
679
 
497
680
  file_key = Cnvrg::Helpers.decrypt(key,iv, f["path"])
498
681
  resp = false
499
- File.open(project_home+"/"+file_path, 'w+') do |file|
500
- resp = client.get_object({bucket:bucket,
682
+ Cnvrg::Helpers.try_until_success(tries: 10) {
683
+ File.open(project_home+"/"+file_path, 'w+') do |file|
684
+ resp = client.get_object({bucket:bucket,
501
685
  key:file_key}, target: file)
502
- #TODO update idx here!
503
- end
504
- if resp
505
- download_succ_count +=1
506
- else
507
- return Cnvrg::Result.new(false,"Could not create file: #{file_path}")
508
- end
686
+ end
687
+ }
688
+ progress.progress += 1 if progress.present?
689
+ download_succ_count += 1
509
690
 
510
691
 
511
692
 
512
- rescue => e
513
- return Cnvrg::Result.new(false,"Could not create file: #{file_path}", e.message, e.backtrace)
514
- raise Parallel::Kill
515
- end
693
+ rescue => e
694
+ return Cnvrg::Result.new(false,"Could not create file: #{file_path}", e.message, e.backtrace)
695
+ raise Parallel::Kill
696
+ end
516
697
 
517
698
 
518
699
 
519
- end
520
700
  end
701
+ end
521
702
  if download_succ_count == files["keys"].size
522
703
  return Cnvrg::Result.new(true,"Done.\nDownloaded #{download_succ_count} files")
523
704
  end
524
705
  rescue => e
525
-
526
706
  return Cnvrg::Result.new(false,"Could not download some files", e.message, e.backtrace)
527
707
  end
528
708
 
@@ -530,8 +710,7 @@ module Cnvrg
530
710
 
531
711
 
532
712
  end
533
-
534
- def download_file(absolute_path, relative_path, project_home, conflict=false)
713
+ def download_file(absolute_path, relative_path, project_home, conflict=false)
535
714
  res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
536
715
  Cnvrg::CLI.is_response_success(res, false)
537
716
  if res["result"]
@@ -658,9 +837,9 @@ module Cnvrg
658
837
  return true
659
838
 
660
839
  end
661
- def start_commit(new_branch,force:false, exp_start_commit:nil, job_slug: nil, job_type: nil, start_commit: nil)
840
+ def start_commit(new_branch,force:false, exp_start_commit:nil, job_slug: nil, job_type: nil, start_commit: nil, message: nil)
662
841
  response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug, new_branch: new_branch,force:force,
663
- username: @owner, exp_start_commit:exp_start_commit, job_slug: job_slug, job_type: job_type, start_commit: start_commit})
842
+ username: @owner, exp_start_commit:exp_start_commit, job_slug: job_slug, job_type: job_type, start_commit: start_commit, message: message})
664
843
  Cnvrg::CLI.is_response_success(response,false)
665
844
  return response
666
845
  end
@@ -680,9 +859,17 @@ module Cnvrg
680
859
  end
681
860
 
682
861
  def delete(file)
862
+ file = "#{@project_home}/#{file}" unless File.exists? file
863
+ return unless File.exists? file
683
864
  FileUtils.rm_rf(file)
684
865
  end
685
866
 
867
+ def delete_conflict(file)
868
+ file = "#{@project_home}/#{file}" unless File.exists? file
869
+ return unless File.exists? file
870
+ File.rename(file, "#{file}.deleted")
871
+ end
872
+
686
873
  def handle_compare_idx(compared, resolver: {})
687
874
  begin
688
875
  all_files = compared.values.flatten.uniq
@@ -710,7 +897,7 @@ module Cnvrg
710
897
  self.download_file(file_path: "#{file}.conflict", key: key, iv: iv, bucket: bucket, path: files[file]['path'], client: client)
711
898
  next
712
899
  end
713
- if compared['updated_on_server'].include? file or compared['added'].include? file
900
+ if compared['updated_on_server'].include? file
714
901
  self.download_file(file_path: file, key: key, iv: iv, bucket: bucket, path: files[file]['path'], client: client)
715
902
  next
716
903
  end
@@ -0,0 +1,75 @@
1
+ module Cnvrg
2
+ class Flows
3
+ def initialize(project_path, fullpath)
4
+ @project = Cnvrg::Project.new(project_path)
5
+ @fullpath = fullpath
6
+ @tasks = {}
7
+ @relations = {}
8
+ @title = nil
9
+ @slug = nil
10
+ @base_resource = @project.base_resource + "flows"
11
+
12
+ self.reload_flow
13
+ end
14
+
15
+ def self.create_flow(path, flow)
16
+ File.open(path, "w"){|file| file.write flow.to_yaml}
17
+ end
18
+
19
+ def get_flow
20
+ unless File.exists? @fullpath
21
+ raise StandardError.new("Cant find flow in #{@fullpath}")
22
+ end
23
+ YAML.load_file(@fullpath)
24
+ end
25
+
26
+ def set_flow(new_flow)
27
+ File.open(@fullpath, "w"){|file| file.write new_flow.to_yaml}
28
+ end
29
+
30
+ def set_flow_slug(slug)
31
+ flow = self.get_flow
32
+ flow[:slug] = slug
33
+ self.set_flow(flow)
34
+ end
35
+
36
+ def reload_flow
37
+ flow = self.get_flow
38
+ @title = flow[:title]
39
+ @slug = flow[:slug]
40
+ @relations = flow[:relations]
41
+ local_tasks = flow[:tasks] || {}
42
+ @relations.each do |relation|
43
+ relation.values.each do |task|
44
+ if local_tasks[task].present?
45
+ @tasks[task] = Cnvrg::Task.new(@project.local_path, content: local_tasks[task])
46
+ else
47
+ @tasks[task] = Cnvrg::Task.new(@project.local_path, path: task)
48
+ end
49
+ end
50
+ end
51
+ end
52
+
53
+ def run
54
+ resp = Cnvrg::API.request(@base_resource, 'POST', {data: to_api})
55
+ Cnvrg::CLI.is_response_success(resp, true)
56
+ flow_slug = resp['result']['flow']
57
+ self.set_flow_slug(flow_slug)
58
+ url = Cnvrg::Helpers.remote_url + resp['result']['url']
59
+ return url
60
+ end
61
+
62
+
63
+ private
64
+ def to_api
65
+ {
66
+ relations: @relations,
67
+ tasks: @tasks.keys.map{|task| [task, @tasks[task].to_api]}.to_h,
68
+ title: @title,
69
+ slug: @slug
70
+ }
71
+ end
72
+
73
+
74
+ end
75
+ end