cnvrg 0.8.5 → 0.8.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +54 -192
- data/lib/cnvrg/files.rb +26 -6
- data/lib/cnvrg/project.rb +265 -257
- data/lib/cnvrg/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 819730717280428ac1ba231d43462cb827253cc1
|
4
|
+
data.tar.gz: 95c1963b178296af0655d54b691b02895ced41ef
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 61563035b65f94b7d87354a6dcb783119179edf655cd6881595035417d15c6441b0337f48ae9de9b410136bd3133eb3254fb995af46ffbd67d785e09312aa3e2
|
7
|
+
data.tar.gz: fda3bfcd38199e6c04c46a2c2ca7812182e58352c0a0c9d06898acca3f5de30178160d35ab6b6d86d640d7c89ecb1a773a562d44af369e652b5e476943aa8c21
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -305,7 +305,7 @@ module Cnvrg
|
|
305
305
|
compression_path = "#{home_dir}/.cnvrg/tmp"
|
306
306
|
config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url, compression_path: compression_path}
|
307
307
|
File.open(home_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
|
308
|
-
say "Done"
|
308
|
+
say "Done", Thor::Shell::Color::GREEN
|
309
309
|
rescue
|
310
310
|
say "ERROR", Thor::Shell::Color::RED
|
311
311
|
File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
@@ -340,7 +340,7 @@ module Cnvrg
|
|
340
340
|
compression_path = "#{home_dir}/.cnvrg/tmp"
|
341
341
|
config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url, compression_path: compression_path}
|
342
342
|
File.open(home_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
|
343
|
-
say "Done"
|
343
|
+
say "Done", Thor::Shell::Color::GREEN
|
344
344
|
rescue
|
345
345
|
say "ERROR", Thor::Shell::Color::RED
|
346
346
|
end
|
@@ -408,7 +408,7 @@ module Cnvrg
|
|
408
408
|
end
|
409
409
|
end
|
410
410
|
rescue SignalException
|
411
|
-
say "\nAborting"
|
411
|
+
say "\nAborting", Thor::Shell::Color::RED
|
412
412
|
exit(1)
|
413
413
|
end
|
414
414
|
end
|
@@ -442,7 +442,7 @@ module Cnvrg
|
|
442
442
|
say "#{checks} Done", Thor::Shell::Color::GREEN
|
443
443
|
|
444
444
|
rescue SignalException
|
445
|
-
say "\nAborting"
|
445
|
+
say "\nAborting", Thor::Shell::Color::RED
|
446
446
|
exit(1)
|
447
447
|
end
|
448
448
|
end
|
@@ -544,7 +544,7 @@ module Cnvrg
|
|
544
544
|
exit(1)
|
545
545
|
rescue SignalException
|
546
546
|
|
547
|
-
say "\nAborting"
|
547
|
+
say "\nAborting",Thor::Shell::Color::RED
|
548
548
|
logout()
|
549
549
|
exit(1)
|
550
550
|
end
|
@@ -565,7 +565,7 @@ module Cnvrg
|
|
565
565
|
puts e.message
|
566
566
|
puts e.backtrace
|
567
567
|
rescue SignalException
|
568
|
-
say "\nAborting"
|
568
|
+
say "\nAborting",Thor::Shell::Color::RED
|
569
569
|
exit(1)
|
570
570
|
end
|
571
571
|
|
@@ -587,7 +587,7 @@ module Cnvrg
|
|
587
587
|
|
588
588
|
|
589
589
|
rescue SignalException
|
590
|
-
say "\nAborting"
|
590
|
+
say "\nAborting", Thor::Shell::Color::RED
|
591
591
|
exit(1)
|
592
592
|
end
|
593
593
|
end
|
@@ -676,7 +676,7 @@ module Cnvrg
|
|
676
676
|
|
677
677
|
@project.revert(working_dir)
|
678
678
|
end
|
679
|
-
say "\nAborting"
|
679
|
+
say "\nAborting", Thor::Shell::Color::RED
|
680
680
|
exit(1)
|
681
681
|
end
|
682
682
|
end
|
@@ -762,7 +762,7 @@ module Cnvrg
|
|
762
762
|
log_error(e)
|
763
763
|
rescue SignalException
|
764
764
|
|
765
|
-
say "\nAborting"
|
765
|
+
say "\nAborting", Thor::Shell::Color::RED
|
766
766
|
exit(1)
|
767
767
|
end
|
768
768
|
end
|
@@ -802,7 +802,7 @@ module Cnvrg
|
|
802
802
|
rescue => e
|
803
803
|
log_error(e)
|
804
804
|
rescue SignalException
|
805
|
-
say "\nAborting"
|
805
|
+
say "\nAborting", Thor::Shell::Color::RED
|
806
806
|
exit(1)
|
807
807
|
end
|
808
808
|
end
|
@@ -826,7 +826,7 @@ module Cnvrg
|
|
826
826
|
log_error(e)
|
827
827
|
rescue SignalException
|
828
828
|
|
829
|
-
say "\nAborting"
|
829
|
+
say "\nAborting", Thor::Shell::Color::RED
|
830
830
|
exit(1)
|
831
831
|
end
|
832
832
|
end
|
@@ -891,6 +891,7 @@ module Cnvrg
|
|
891
891
|
downloaded_files += files['keys'].length
|
892
892
|
files = @files.get_clone_chunk(commit: commit, latest_id: files['latest'])
|
893
893
|
end
|
894
|
+
progressbar.finish
|
894
895
|
if downloaded_files == files_count
|
895
896
|
Dataset.verify_cnvrgignore_exist(dataset_name, false)
|
896
897
|
@dataset = Dataset.new(dataset_home)
|
@@ -905,7 +906,7 @@ module Cnvrg
|
|
905
906
|
exit(1)
|
906
907
|
end
|
907
908
|
rescue SignalException
|
908
|
-
say "\nAborting"
|
909
|
+
say "\nAborting", Thor::Shell::Color::RED
|
909
910
|
exit(1)
|
910
911
|
end
|
911
912
|
end
|
@@ -983,7 +984,7 @@ module Cnvrg
|
|
983
984
|
end
|
984
985
|
end
|
985
986
|
rescue SignalException
|
986
|
-
say "\nAborting"
|
987
|
+
say "\nAborting", Thor::Shell::Color::RED
|
987
988
|
exit(1)
|
988
989
|
end
|
989
990
|
end
|
@@ -1009,7 +1010,7 @@ module Cnvrg
|
|
1009
1010
|
|
1010
1011
|
rescue SignalException
|
1011
1012
|
|
1012
|
-
say "\nAborting"
|
1013
|
+
say "\nAborting", Thor::Shell::Color::RED
|
1013
1014
|
exit(1)
|
1014
1015
|
end
|
1015
1016
|
end
|
@@ -1031,7 +1032,7 @@ module Cnvrg
|
|
1031
1032
|
rescue SignalException
|
1032
1033
|
log_end(-1)
|
1033
1034
|
|
1034
|
-
say "\nAborting"
|
1035
|
+
say "\nAborting", Thor::Shell::Color::RED
|
1035
1036
|
exit(1)
|
1036
1037
|
end
|
1037
1038
|
end
|
@@ -1813,112 +1814,22 @@ module Cnvrg
|
|
1813
1814
|
|
1814
1815
|
if clone_resp
|
1815
1816
|
@project = Project.new(project_home)
|
1816
|
-
@files = Cnvrg::Files.new(@project.owner, slug)
|
1817
|
+
@files = Cnvrg::Files.new(@project.owner, slug, project_home: project_home)
|
1817
1818
|
response = @project.clone(remote, commit_to_clone)
|
1818
1819
|
Cnvrg::CLI.is_response_success response
|
1820
|
+
commit_sha1 = response["result"]["commit"]
|
1821
|
+
files = response["result"]["tree"].keys
|
1819
1822
|
idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
|
1820
|
-
|
1821
1823
|
log_message("Downloading files", Thor::Shell::Color::BLUE)
|
1822
|
-
|
1823
|
-
|
1824
|
-
|
1825
|
-
|
1826
|
-
|
1827
|
-
|
1828
|
-
|
1829
|
-
rescue Interrupt
|
1830
|
-
log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
|
1831
|
-
|
1832
|
-
@files.revoke_clone(project_home)
|
1833
|
-
return
|
1834
|
-
end
|
1835
|
-
Project.verify_cnvrgignore_exist(project_name, remote)
|
1836
|
-
if !download_result.is_success?
|
1837
|
-
log_message(download_result.e_msg, download_result.msg_color, false)
|
1838
|
-
log_message(download_result.e_msg_backtrace, download_result.msg_color, false)
|
1839
|
-
|
1840
|
-
@files.revoke_clone(project_home)
|
1841
|
-
end
|
1842
|
-
log_message(download_result.msg, download_result.msg_color)
|
1843
|
-
else
|
1844
|
-
current_commit = response["result"]["commit"]
|
1845
|
-
if !response["result"]["tree"].nil?
|
1846
|
-
parallel_options = {
|
1847
|
-
:progress => {
|
1848
|
-
:title => "Download Progress",
|
1849
|
-
:progress_mark => '=',
|
1850
|
-
:format => "%b>>%i| %p%% %t",
|
1851
|
-
:starting_at => 0,
|
1852
|
-
:total => response["result"]["tree"].size,
|
1853
|
-
:autofinish => true
|
1854
|
-
},
|
1855
|
-
in_processes: ParallelProcesses,
|
1856
|
-
in_thread: ParallelThreads
|
1857
|
-
}
|
1858
|
-
begin
|
1859
|
-
successful_changes =[]
|
1860
|
-
is_success = true
|
1861
|
-
clone_result = Parallel.map((response["result"]["tree"]), parallel_options) do |f|
|
1862
|
-
|
1863
|
-
relative_path = f[0].gsub(/^#{@project.local_path}/, "")
|
1864
|
-
if f[0].end_with? "/"
|
1865
|
-
# dir
|
1866
|
-
if @files.download_dir(f[0], relative_path, project_home)
|
1867
|
-
f
|
1868
|
-
successful_changes << relative_path
|
1869
|
-
else
|
1870
|
-
is_success =false
|
1871
|
-
log_message("Could not create directory: #{f[0]}", Thor::Shell::Color::RED)
|
1872
|
-
raise Parallel::Kill
|
1873
|
-
end
|
1874
|
-
else
|
1875
|
-
# blob
|
1876
|
-
|
1877
|
-
if @files.download_file_s3(f[0], relative_path, project_home, commit_sha1=current_commit)
|
1878
|
-
f
|
1879
|
-
successful_changes << relative_path
|
1880
|
-
else
|
1881
|
-
is_success =false
|
1882
|
-
log_message("Could not download file: #{f[0]}", Thor::Shell::Color::RED)
|
1883
|
-
raise Parallel::Kill
|
1884
|
-
|
1885
|
-
end
|
1886
|
-
end
|
1887
|
-
end
|
1888
|
-
successful_changes = response["result"]["tree"]
|
1889
|
-
if !successful_changes.nil? and is_success
|
1890
|
-
@project.set_idx(idx)
|
1891
|
-
Project.verify_cnvrgignore_exist(project_name,remote)
|
1892
|
-
log_message("Done.\nDownloaded #{successful_changes.size}/#{response["result"]["tree"].size} files", Thor::Shell::Color::GREEN)
|
1893
|
-
else
|
1894
|
-
log_message("Couldn't download some files", Thor::Shell::Color::RED)
|
1895
|
-
|
1896
|
-
end
|
1897
|
-
rescue Interrupt
|
1898
|
-
log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
|
1899
|
-
|
1900
|
-
@files.revoke_clone(project_home)
|
1901
|
-
return
|
1902
|
-
end
|
1903
|
-
end
|
1904
|
-
|
1905
|
-
|
1906
|
-
end
|
1907
|
-
|
1908
|
-
else
|
1909
|
-
|
1910
|
-
log_message("Error: Couldn't clone: #{project_name}", Thor::Shell::Color::RED)
|
1911
|
-
@files.revoke_clone(project_home)
|
1912
|
-
|
1913
|
-
return
|
1824
|
+
progressbar = @files.create_progressbar(files.size, "Clone Progress")
|
1825
|
+
@files.download_files(files, commit_sha1, progress: progressbar)
|
1826
|
+
progressbar.finish
|
1827
|
+
Project.verify_cnvrgignore_exist(project_name, remote)
|
1828
|
+
@project.set_idx(idx)
|
1829
|
+
log_message("Done")
|
1830
|
+
log_message("Downloaded #{files.size} files")
|
1914
1831
|
end
|
1915
|
-
rescue SignalException
|
1916
|
-
say "\nAborting"
|
1917
|
-
@files.revoke_clone(project_home)
|
1918
|
-
|
1919
|
-
return
|
1920
1832
|
end
|
1921
|
-
|
1922
1833
|
end
|
1923
1834
|
|
1924
1835
|
|
@@ -2360,7 +2271,7 @@ module Cnvrg
|
|
2360
2271
|
:autofinish => true)
|
2361
2272
|
@files.upload_multiple_files(to_upload, commit_sha1, progress: progressbar)
|
2362
2273
|
@files.delete_files_from_server(deleted, commit_sha1)
|
2363
|
-
progressbar.
|
2274
|
+
progressbar.finish
|
2364
2275
|
res = @files.end_commit(commit_sha1, force: force, message: commit_msg)
|
2365
2276
|
unless Cnvrg::CLI.is_response_success(res, false)
|
2366
2277
|
raise StandardError.new("Cant end commit")
|
@@ -2614,6 +2525,9 @@ module Cnvrg
|
|
2614
2525
|
if git or @project.is_git
|
2615
2526
|
return download_in_git(commit)
|
2616
2527
|
end
|
2528
|
+
if commit.present?
|
2529
|
+
return jump(commit)
|
2530
|
+
end
|
2617
2531
|
ignore = options[:ignore] || ""
|
2618
2532
|
if ignore.nil? or ignore.empty?
|
2619
2533
|
ignore = ignore_list
|
@@ -2683,17 +2597,20 @@ module Cnvrg
|
|
2683
2597
|
|
2684
2598
|
|
2685
2599
|
Cnvrg::Logger.log_info("Downloading updated files:#{updated_files.join(",")}")
|
2686
|
-
@files.download_files(updated_files, commit, progress: progressbar)
|
2600
|
+
@files.download_files(updated_files, commit, progress: progressbar)
|
2687
2601
|
|
2688
2602
|
Cnvrg::Logger.log_info("Downloading conflicted files:#{conflicted_files.join(",")}")
|
2689
|
-
@files.download_files(conflicted_files, commit, postfix: ".conflict", progress: progressbar)
|
2603
|
+
@files.download_files(conflicted_files, commit, postfix: ".conflict", progress: progressbar)
|
2690
2604
|
|
2691
2605
|
Cnvrg::Logger.log_info("Delete files: #{deleted_files.join(",")}")
|
2692
2606
|
@files.delete_files_local(deleted_files, conflicted: conflicted_deleted, progress: progressbar)
|
2693
2607
|
|
2694
2608
|
# update idx with latest commit
|
2695
|
-
|
2609
|
+
# the latest true its because if we define --commit in the cmd it will go to "def jump(options['commit'])"
|
2610
|
+
# so if we are downloads something, we have to stay here.
|
2611
|
+
@project.update_idx_with_commit!(commit, latest: true)
|
2696
2612
|
#TODO Sync, remove idx, sync again and pray
|
2613
|
+
progressbar.finish
|
2697
2614
|
check = Helpers.checkmark()
|
2698
2615
|
Cnvrg::Logger.log_info("Finished downloading successfuly")
|
2699
2616
|
if options["verbose"]
|
@@ -2728,10 +2645,11 @@ module Cnvrg
|
|
2728
2645
|
project_home = get_project_home
|
2729
2646
|
@project = Project.new(project_home)
|
2730
2647
|
current_commit = @project.last_local_commit
|
2731
|
-
if current_commit.
|
2648
|
+
if current_commit.start_with? commit_sha1 #commit_sha1 can be partial.
|
2732
2649
|
log_message("Project is already updated", Thor::Shell::Color::GREEN)
|
2733
2650
|
exit(0)
|
2734
2651
|
end
|
2652
|
+
log_message("Jumping to commit #{commit_sha1}")
|
2735
2653
|
@files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: project_home)
|
2736
2654
|
resp = @project.jump_idx(destination: commit_sha1)
|
2737
2655
|
if resp.blank?
|
@@ -2739,81 +2657,27 @@ module Cnvrg
|
|
2739
2657
|
exit(0)
|
2740
2658
|
end
|
2741
2659
|
compare = resp['result']['compare']
|
2742
|
-
resolver = resp['result']['resolver']
|
2743
2660
|
latest = resp['result']['latest']
|
2744
2661
|
commit = resp['result']['commit']
|
2745
|
-
updated_files = compare['updated_on_server']
|
2662
|
+
updated_files = compare['updated_on_server'] + compare['added']
|
2746
2663
|
conflicted_files = compare['conflicts']
|
2664
|
+
conflicted_deleted = compare['delete_conflicts'] || []
|
2747
2665
|
deleted_files = compare['deleted']
|
2748
2666
|
overall_changes = [updated_files, conflicted_files, deleted_files].flatten.size
|
2749
2667
|
|
2750
|
-
|
2751
|
-
progressbar = ProgressBar.create(:title => "Download Progress",
|
2752
|
-
:progress_mark => '=',
|
2753
|
-
:format => "%b>>%i| %p%% %t",
|
2754
|
-
:starting_at => 0,
|
2755
|
-
:total => overall_changes,
|
2756
|
-
:autofinish => true)
|
2757
|
-
|
2668
|
+
progressbar = @files.create_progressbar(overall_changes, "Download Progress")
|
2758
2669
|
@files.download_files(updated_files, commit_sha1, progress: progressbar)
|
2759
2670
|
@files.download_files(conflicted_files, commit_sha1, progress: progressbar, postfix: '.conflicted')
|
2760
|
-
@files.delete_files_local(deleted_files, progress: progressbar)
|
2761
|
-
|
2762
|
-
|
2763
|
-
@project.set_on_branch(latest)
|
2764
|
-
@project.update_idx_with_commit!(commit)
|
2671
|
+
@files.delete_files_local(deleted_files, progress: progressbar, conflicted: conflicted_deleted)
|
2672
|
+
progressbar.finish
|
2673
|
+
@project.update_idx_with_commit!(commit, latest: latest)
|
2765
2674
|
@project.generate_idx
|
2675
|
+
|
2766
2676
|
log_message("Jumped successfuly!", Thor::Shell::Color::GREEN)
|
2767
2677
|
rescue => e
|
2678
|
+
Logger::log_error(e)
|
2768
2679
|
log_message("Cant jump to the specified commit", Thor::Shell::Color::RED)
|
2769
2680
|
end
|
2770
|
-
# successful_changes = []
|
2771
|
-
#
|
2772
|
-
# if !response["result"]["tree"].nil?
|
2773
|
-
|
2774
|
-
# commit_sha1 = response["result"]["commit"]
|
2775
|
-
# idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
|
2776
|
-
# File.open(project_home + "/.cnvrg/idx.yml", "w+") {|f| f.write idx.to_yaml}
|
2777
|
-
# if is_remote
|
2778
|
-
# current_tree = Dir.glob("**/*", File::FNM_DOTMATCH).flatten.reject {|file| file.start_with? '.' or file.eql? "__init__.py" or file.eql? "uwsgi.ini" or file.ends_with? "/." or file.eql? "."}
|
2779
|
-
# else
|
2780
|
-
# current_tree = Dir.glob("**/*", File::FNM_DOTMATCH).flatten.reject {|file| file.start_with? '.cnvrg' or file.ends_with? "/." or file.eql? "."}
|
2781
|
-
# end
|
2782
|
-
#
|
2783
|
-
# jump_result = Parallel.map(response["result"]["tree"], parallel_options) do |f|
|
2784
|
-
#
|
2785
|
-
# relative_path = f[0].gsub(/^#{@project.local_path}/, "")
|
2786
|
-
# log_message("Downloading #{f[0]}", Thor::Shell::Color::BLUE, false)
|
2787
|
-
# if f[0].end_with? "/"
|
2788
|
-
# # dir
|
2789
|
-
# @files.download_dir(f[0], relative_path, project_home)
|
2790
|
-
#
|
2791
|
-
# else
|
2792
|
-
# # blob
|
2793
|
-
# @files.download_file_s3(f[0], relative_path, project_home, commit_sha1 = commit_sha1)
|
2794
|
-
# end
|
2795
|
-
# end
|
2796
|
-
#
|
2797
|
-
#
|
2798
|
-
# successful_changes = jump_result.select {|x| not x.nil?}
|
2799
|
-
# end
|
2800
|
-
# response["result"]["tree"].each do |f|
|
2801
|
-
# if f[0].end_with? "/"
|
2802
|
-
# current_tree.delete(f[0][0, f[0].size - 1])
|
2803
|
-
# else
|
2804
|
-
# current_tree.delete(f[0])
|
2805
|
-
# end
|
2806
|
-
# end
|
2807
|
-
#
|
2808
|
-
# FileUtils.rm_rf(current_tree)
|
2809
|
-
# log_message("Done. Jumped to #{commit_sha1} completed successfully", Thor::Shell::Color::GREEN)
|
2810
|
-
# rescue => e
|
2811
|
-
# log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
|
2812
|
-
# log_error(e)
|
2813
|
-
# rescue SignalException
|
2814
|
-
#
|
2815
|
-
# exit(1)
|
2816
|
-
# end
|
2817
2681
|
end
|
2818
2682
|
|
2819
2683
|
desc 'show', 'Show specific file from a specific commit'
|
@@ -2911,6 +2775,7 @@ module Cnvrg
|
|
2911
2775
|
method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
|
2912
2776
|
def sync(direct = true)
|
2913
2777
|
verify_logged_in(true) if direct
|
2778
|
+
@project = Project.new(get_project_home)
|
2914
2779
|
log_start(__method__, args, options)
|
2915
2780
|
log_message('Checking for new updates from remote version', Thor::Shell::Color::BLUE, options["verbose"])
|
2916
2781
|
log_message('Syncing project', Thor::Shell::Color::BLUE, !options["verbose"])
|
@@ -2919,7 +2784,7 @@ module Cnvrg
|
|
2919
2784
|
in_exp = options["in_exp"] || (job_slug.present? and job_type.present?)
|
2920
2785
|
in_exp = false if job_type.present? and job_type == "NotebookSession"
|
2921
2786
|
run_download = true
|
2922
|
-
if options[:force] or options[:files].present? or options[:output_dir].present? or in_exp
|
2787
|
+
if options[:force] or options[:files].present? or options[:output_dir].present? or in_exp or @project.is_branch
|
2923
2788
|
run_download = false
|
2924
2789
|
end
|
2925
2790
|
if run_download
|
@@ -3477,10 +3342,6 @@ module Cnvrg
|
|
3477
3342
|
else
|
3478
3343
|
sync_result = invoke :sync, [false], :force => false
|
3479
3344
|
end
|
3480
|
-
|
3481
|
-
|
3482
|
-
|
3483
|
-
|
3484
3345
|
end
|
3485
3346
|
#handle grid if it's git project
|
3486
3347
|
if project.is_git and grid.present?
|
@@ -3516,6 +3377,7 @@ module Cnvrg
|
|
3516
3377
|
if forced_commit and (commit_to_run.nil? or commit_to_run.empty?)
|
3517
3378
|
commit_to_run = forced_commit
|
3518
3379
|
end
|
3380
|
+
|
3519
3381
|
commit_to_run = commit_to_run.presence || project.last_local_commit
|
3520
3382
|
|
3521
3383
|
res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
|
@@ -5491,7 +5353,7 @@ module Cnvrg
|
|
5491
5353
|
end
|
5492
5354
|
end
|
5493
5355
|
|
5494
|
-
def log_message(message, type, to_print = true)
|
5356
|
+
def log_message(message, type=Thor::Shell::Color::GREEN, to_print = true)
|
5495
5357
|
if to_print
|
5496
5358
|
say message, type
|
5497
5359
|
end
|
@@ -5538,10 +5400,10 @@ module Cnvrg
|
|
5538
5400
|
error = response['message'] || "Unknown error"
|
5539
5401
|
# Cnvrg::CLI.log_end(1, error)
|
5540
5402
|
if response["status"] == 500
|
5541
|
-
|
5403
|
+
log_message('Server Error', Thor::Shell::Color::RED)
|
5542
5404
|
else
|
5543
5405
|
$LOG.error message: error, type: "error"
|
5544
|
-
|
5406
|
+
log_message("error", Thor::Shell::Color::RED)
|
5545
5407
|
end
|
5546
5408
|
|
5547
5409
|
if should_exit
|
@@ -5555,7 +5417,7 @@ module Cnvrg
|
|
5555
5417
|
puts e.message
|
5556
5418
|
puts e.backtrace
|
5557
5419
|
rescue SignalException
|
5558
|
-
|
5420
|
+
log_message "Aborting", Thor::Shell::Color::RED
|
5559
5421
|
end
|
5560
5422
|
|
5561
5423
|
end
|
@@ -5837,7 +5699,7 @@ module Cnvrg
|
|
5837
5699
|
rescue
|
5838
5700
|
return false
|
5839
5701
|
rescue SignalException
|
5840
|
-
say "\nAborting"
|
5702
|
+
say "\nAborting", Thor::Shell::Color::RED
|
5841
5703
|
Exit(0)
|
5842
5704
|
end
|
5843
5705
|
|
data/lib/cnvrg/files.rb
CHANGED
@@ -567,17 +567,15 @@ module Cnvrg
|
|
567
567
|
end
|
568
568
|
|
569
569
|
|
570
|
-
def download_file_s3(
|
570
|
+
def download_file_s3(relative_path, commit_sha1=nil, postfix: '')
|
571
571
|
begin
|
572
|
-
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {
|
572
|
+
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {relative_path: relative_path,
|
573
573
|
commit_sha1: commit_sha1,new_version:true})
|
574
574
|
|
575
575
|
Cnvrg::CLI.is_response_success(res, false)
|
576
576
|
if res["result"]
|
577
577
|
download_resp = res
|
578
578
|
filename = download_resp["result"]["filename"]
|
579
|
-
|
580
|
-
absolute_path += ".conflict" if conflict
|
581
579
|
sts_path = download_resp["result"]["path_sts"]
|
582
580
|
retries = 0
|
583
581
|
success= false
|
@@ -632,8 +630,8 @@ module Cnvrg
|
|
632
630
|
:endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
|
633
631
|
:http_open_timeout => 60, :retry_limit => 20)
|
634
632
|
end
|
635
|
-
|
636
|
-
File.open(
|
633
|
+
absolute_path = "#{@project_home}/#{relative_path}#{postfix}"
|
634
|
+
File.open(absolute_path, 'w+') do |file|
|
637
635
|
resp = client.get_object({bucket:bucket,
|
638
636
|
key:file_key}, target: file)
|
639
637
|
end
|
@@ -649,7 +647,21 @@ module Cnvrg
|
|
649
647
|
end
|
650
648
|
end
|
651
649
|
|
650
|
+
def create_progressbar(length = 10, title = 'Progress')
|
651
|
+
ProgressBar.create(:title => title,
|
652
|
+
:progress_mark => '=',
|
653
|
+
:format => "%b>>%i| %p%% %t",
|
654
|
+
:starting_at => 0,
|
655
|
+
:total => length,
|
656
|
+
:autofinish => true)
|
657
|
+
end
|
658
|
+
|
652
659
|
def download_files(files, commit, postfix: '', progress: nil)
|
660
|
+
return if files.blank?
|
661
|
+
if Cnvrg::Helpers.server_version < 1
|
662
|
+
Cnvrg::Logger.log_info("Download files from older server.")
|
663
|
+
return self.download_files_old(files, commit, progress: progress, postfix: postfix)
|
664
|
+
end
|
653
665
|
res = Cnvrg::API.request(@base_resource + "download_files", 'POST', {files: files, commit: commit})
|
654
666
|
unless Cnvrg::CLI.is_response_success(res, false)
|
655
667
|
raise Exception.new("Cant download files from the server.")
|
@@ -657,6 +669,14 @@ module Cnvrg
|
|
657
669
|
self.download_multpile_files_s3(res['result'], @project_home, postfix: postfix, progress: progress)
|
658
670
|
end
|
659
671
|
|
672
|
+
|
673
|
+
def download_files_old(files, commit, postfix: '', progress: nil)
|
674
|
+
files.each do |file|
|
675
|
+
self.download_file_s3(file, commit, postfix: postfix)
|
676
|
+
progress.progress += 1 if progress.present?
|
677
|
+
end
|
678
|
+
end
|
679
|
+
|
660
680
|
def delete_files_local(deleted, conflicted: [], progress: nil)
|
661
681
|
deleted -= conflicted
|
662
682
|
deleted.each{|file| self.delete(file); progress.progress += 1 if progress.present?}
|
data/lib/cnvrg/project.rb
CHANGED
@@ -11,7 +11,7 @@ module Cnvrg
|
|
11
11
|
begin
|
12
12
|
@local_path = project_home
|
13
13
|
@working_dir = project_home
|
14
|
-
config = YAML.load_file(project_home+"/.cnvrg/config.yml")
|
14
|
+
config = YAML.load_file(project_home + "/.cnvrg/config.yml")
|
15
15
|
@title = config[:project_name]
|
16
16
|
@slug = config[:project_slug]
|
17
17
|
@owner = config[:owner]
|
@@ -45,7 +45,7 @@ module Cnvrg
|
|
45
45
|
end
|
46
46
|
list = new_ignore.split(",")
|
47
47
|
begin
|
48
|
-
File.open(self.local_path+"/.cnvrgignore", "a+") do |f|
|
48
|
+
File.open(self.local_path + "/.cnvrgignore", "a+") do |f|
|
49
49
|
f.puts("\n")
|
50
50
|
list.each do |i|
|
51
51
|
f.puts("#{i}\n")
|
@@ -59,16 +59,16 @@ module Cnvrg
|
|
59
59
|
|
60
60
|
def get_ignore_list
|
61
61
|
ignore_list = []
|
62
|
-
if !File.exist? self.local_path+"/.cnvrgignore"
|
62
|
+
if !File.exist? self.local_path + "/.cnvrgignore"
|
63
63
|
return ignore_list
|
64
64
|
end
|
65
|
-
File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
|
65
|
+
File.open(self.local_path + "/.cnvrgignore", "r").each_line do |line|
|
66
66
|
line = line.strip
|
67
67
|
if line.start_with? "#" or ignore_list.include? line or line.empty?
|
68
68
|
next
|
69
69
|
end
|
70
70
|
if line.ends_with? "*"
|
71
|
-
list_regex = Dir.glob("**/#{line}",File::FNM_DOTMATCH).flatten
|
71
|
+
list_regex = Dir.glob("**/#{line}", File::FNM_DOTMATCH).flatten
|
72
72
|
list_regex.each do |l|
|
73
73
|
ignore_list << l
|
74
74
|
if File.directory?(l)
|
@@ -79,15 +79,15 @@ module Cnvrg
|
|
79
79
|
|
80
80
|
end
|
81
81
|
elsif line.ends_with? "/*"
|
82
|
-
line = line.gsub("/*","")
|
82
|
+
line = line.gsub("/*", "")
|
83
83
|
regex_list = Dir.glob("**/#{line}/**/*", File::FNM_DOTMATCH).flatten
|
84
84
|
ignore_list << regex_list
|
85
85
|
elsif line.include? "*"
|
86
|
-
|
87
|
-
|
86
|
+
regex_list = Dir.glob("**/#{line}").flatten
|
87
|
+
ignore_list << regex_list
|
88
88
|
elsif line.end_with? "/" or File.directory?(line)
|
89
|
-
|
90
|
-
|
89
|
+
ignore_list << line
|
90
|
+
all_sub = Dir.glob("#{line}/**/*", File::FNM_DOTMATCH).flatten
|
91
91
|
|
92
92
|
ignore_list << all_sub.flatten
|
93
93
|
|
@@ -102,16 +102,16 @@ module Cnvrg
|
|
102
102
|
def send_ignore_list()
|
103
103
|
begin
|
104
104
|
ignore_list = []
|
105
|
-
File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
|
105
|
+
File.open(self.local_path + "/.cnvrgignore", "r").each_line do |line|
|
106
106
|
line = line.strip
|
107
107
|
if line.start_with? "#" or ignore_list.include? line or line.empty?
|
108
108
|
next
|
109
109
|
end
|
110
110
|
if line.end_with? "/"
|
111
|
-
ignore_list << line.gsub("/","")
|
112
|
-
ignore_list << line+"."
|
111
|
+
ignore_list << line.gsub("/", "")
|
112
|
+
ignore_list << line + "."
|
113
113
|
elsif line.include? "*"
|
114
|
-
line = line.gsub("*",".*")
|
114
|
+
line = line.gsub("*", ".*")
|
115
115
|
ignore_list << line
|
116
116
|
else
|
117
117
|
ignore_list << line
|
@@ -126,7 +126,7 @@ module Cnvrg
|
|
126
126
|
|
127
127
|
# Create project
|
128
128
|
|
129
|
-
def self.create(project_name, clean, with_docker=false)
|
129
|
+
def self.create(project_name, clean, with_docker = false)
|
130
130
|
if clean
|
131
131
|
list_dirs = [project_name, project_name + "/.cnvrg"]
|
132
132
|
else
|
@@ -149,7 +149,7 @@ module Cnvrg
|
|
149
149
|
]
|
150
150
|
cnvrgreadme = Helpers.readme_content
|
151
151
|
cnvrgignore = Helpers.cnvrgignore_content
|
152
|
-
cnvrghyper
|
152
|
+
cnvrghyper = Helpers.hyper_content
|
153
153
|
|
154
154
|
begin
|
155
155
|
|
@@ -166,10 +166,10 @@ module Cnvrg
|
|
166
166
|
FileUtils.mkdir_p list_dirs
|
167
167
|
FileUtils.touch list_files
|
168
168
|
|
169
|
-
File.open(project_name + "/.cnvrg/config.yml", "w+") {
|
170
|
-
File.open(project_name + "/.cnvrgignore", "w+") {
|
171
|
-
File.open(project_name + "/README.md", "w+") {
|
172
|
-
File.open(project_name + "/src/hyper.yaml", "w+") {
|
169
|
+
File.open(project_name + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
|
170
|
+
File.open(project_name + "/.cnvrgignore", "w+") {|f| f.write cnvrgignore}
|
171
|
+
File.open(project_name + "/README.md", "w+") {|f| f.write cnvrgreadme}
|
172
|
+
File.open(project_name + "/src/hyper.yaml", "w+") {|f| f.write cnvrghyper}
|
173
173
|
|
174
174
|
rescue
|
175
175
|
return false
|
@@ -177,75 +177,76 @@ module Cnvrg
|
|
177
177
|
return true
|
178
178
|
end
|
179
179
|
|
180
|
-
def self.link(owner, project_name, docker=false, git = false)
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
180
|
+
def self.link(owner, project_name, docker = false, git = false)
|
181
|
+
ignore_exits = File.exist? ".cnvrgignore"
|
182
|
+
list_dirs = [".cnvrg"
|
183
|
+
]
|
184
|
+
list_files = [
|
185
|
+
".cnvrg/config.yml"
|
186
|
+
]
|
187
|
+
if !ignore_exits
|
188
|
+
list_files <<
|
189
|
+
".cnvrgignore"
|
190
|
+
end
|
191
191
|
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
config = {project_name: project_name,
|
201
|
-
project_slug: project_slug,
|
202
|
-
owner: owner,
|
203
|
-
git: git}
|
204
|
-
FileUtils.mkdir_p list_dirs
|
205
|
-
FileUtils.touch list_files
|
206
|
-
File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
207
|
-
File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless ignore_exits
|
208
|
-
if !File.exist? "README" and !File.exist? "README.md"
|
209
|
-
FileUtils.touch [ "README.md" ]
|
210
|
-
File.open("README.md", "w+") { |f| f.write cnvrgreadme }
|
211
|
-
end
|
192
|
+
cnvrgreadme = Helpers.readme_content
|
193
|
+
cnvrgignore = Helpers.cnvrgignore_content
|
194
|
+
begin
|
195
|
+
response = Cnvrg::API.request("cli/create_project", 'POST', {title: project_name, owner: owner, is_docker: docker})
|
196
|
+
Cnvrg::CLI.is_response_success(response)
|
197
|
+
response = JSON.parse response["result"]
|
198
|
+
project_slug = response["slug"]
|
212
199
|
|
213
|
-
|
214
|
-
|
215
|
-
|
200
|
+
config = {project_name: project_name,
|
201
|
+
project_slug: project_slug,
|
202
|
+
owner: owner,
|
203
|
+
git: git}
|
204
|
+
FileUtils.mkdir_p list_dirs
|
205
|
+
FileUtils.touch list_files
|
206
|
+
File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
|
207
|
+
File.open(".cnvrgignore", "w+") {|f| f.write cnvrgignore} unless ignore_exits
|
208
|
+
if !File.exist? "README" and !File.exist? "README.md"
|
209
|
+
FileUtils.touch ["README.md"]
|
210
|
+
File.open("README.md", "w+") {|f| f.write cnvrgreadme}
|
216
211
|
end
|
217
|
-
|
212
|
+
|
213
|
+
rescue => e
|
214
|
+
puts e
|
215
|
+
return false
|
218
216
|
end
|
217
|
+
return true
|
218
|
+
end
|
219
219
|
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
220
|
+
def self.clone_dir(project_slug, project_owner, project_name, is_git = false)
|
221
|
+
list_dirs = [project_name,
|
222
|
+
project_name + "/.cnvrg"
|
223
|
+
]
|
224
224
|
|
225
225
|
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
226
|
+
list_files = [
|
227
|
+
project_name + "/.cnvrg/config.yml",
|
228
|
+
project_name + "/.cnvrgignore",
|
229
|
+
]
|
230
|
+
begin
|
231
|
+
config = {project_name: project_name,
|
232
|
+
project_slug: project_slug,
|
233
|
+
owner: project_owner,
|
234
|
+
git: is_git}
|
235
|
+
FileUtils.mkdir_p list_dirs
|
236
|
+
FileUtils.touch list_files
|
237
|
+
cnvrgignore = Helpers.cnvrgignore_content
|
238
238
|
|
239
239
|
|
240
|
-
|
241
|
-
|
240
|
+
File.open(project_name + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
|
241
|
+
File.open(project_name + "/.cnvrgignore", "w+") {|f| f.write cnvrgignore}
|
242
242
|
|
243
|
-
|
244
|
-
|
245
|
-
end
|
246
|
-
return true
|
243
|
+
rescue
|
244
|
+
return false
|
247
245
|
end
|
248
|
-
|
246
|
+
return true
|
247
|
+
end
|
248
|
+
|
249
|
+
def self.verify_cnvrgignore_exist(project_name, remote)
|
249
250
|
path = ".cnvrgignore"
|
250
251
|
if !File.exist? path
|
251
252
|
path = "#{project_name}/.cnvrgignore"
|
@@ -255,10 +256,10 @@ module Cnvrg
|
|
255
256
|
begin
|
256
257
|
list_files = [
|
257
258
|
path
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
259
|
+
]
|
260
|
+
FileUtils.touch list_files
|
261
|
+
cnvrgignore = Helpers.cnvrgignore_content
|
262
|
+
File.open(path, "w+") {|f| f.write cnvrgignore}
|
262
263
|
rescue => e
|
263
264
|
return false
|
264
265
|
end
|
@@ -266,64 +267,66 @@ module Cnvrg
|
|
266
267
|
end
|
267
268
|
end
|
268
269
|
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
270
|
+
def self.clone_dir_remote(project_slug, project_owner, project_name, is_git = false)
|
271
|
+
list_dirs = [
|
272
|
+
".cnvrg"
|
273
|
+
]
|
275
274
|
|
276
|
-
list_files = [
|
277
|
-
".cnvrg/config.yml",
|
278
275
|
|
279
|
-
|
276
|
+
list_files = [
|
277
|
+
".cnvrg/config.yml",
|
280
278
|
|
279
|
+
]
|
281
280
|
|
282
|
-
begin
|
283
|
-
config = {project_name: project_name,
|
284
|
-
project_slug: project_slug,
|
285
|
-
owner: project_owner,
|
286
|
-
git: is_git
|
287
|
-
}
|
288
|
-
FileUtils.mkdir_p list_dirs
|
289
|
-
FileUtils.touch list_files
|
290
281
|
|
282
|
+
begin
|
283
|
+
config = {project_name: project_name,
|
284
|
+
project_slug: project_slug,
|
285
|
+
owner: project_owner,
|
286
|
+
git: is_git
|
287
|
+
}
|
288
|
+
FileUtils.mkdir_p list_dirs
|
289
|
+
FileUtils.touch list_files
|
291
290
|
|
292
|
-
File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
293
|
-
if !File.exist? ".cnvrgignore"
|
294
|
-
FileUtils.touch ".cnvrgignore"
|
295
|
-
list_files << ".cnvrgignore"
|
296
|
-
cnvrgignore = Helpers.cnvrgignore_content
|
297
|
-
File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore }
|
298
291
|
|
292
|
+
File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
|
293
|
+
if !File.exist? ".cnvrgignore"
|
294
|
+
FileUtils.touch ".cnvrgignore"
|
295
|
+
list_files << ".cnvrgignore"
|
296
|
+
cnvrgignore = Helpers.cnvrgignore_content
|
297
|
+
File.open(".cnvrgignore", "w+") {|f| f.write cnvrgignore}
|
299
298
|
|
300
|
-
end
|
301
299
|
|
302
|
-
rescue
|
303
|
-
return false
|
304
300
|
end
|
305
|
-
|
301
|
+
|
302
|
+
rescue
|
303
|
+
return false
|
306
304
|
end
|
305
|
+
return true
|
306
|
+
end
|
307
|
+
|
307
308
|
def update_is_new_branch(new_branch)
|
308
|
-
config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
|
309
|
+
config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
|
309
310
|
config[:new_branch] = new_branch
|
310
|
-
File.open(@working_dir+"/.cnvrg/config.yml", "w+") {
|
311
|
+
File.open(@working_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
|
311
312
|
end
|
313
|
+
|
312
314
|
def get_new_branch
|
313
315
|
begin
|
314
|
-
|
315
|
-
|
316
|
-
rescue =>e
|
316
|
+
config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
|
317
|
+
return config[:new_branch]
|
318
|
+
rescue => e
|
317
319
|
return false
|
318
320
|
|
319
321
|
end
|
320
322
|
end
|
321
323
|
|
322
324
|
def remove_new_branch
|
323
|
-
config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
|
325
|
+
config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
|
324
326
|
new_config = config.except(:new_branch)
|
325
|
-
File.open(@working_dir+"/.cnvrg/config.yml", "w+") {
|
327
|
+
File.open(@working_dir + "/.cnvrg/config.yml", "w+") {|f| f.write new_config.to_yaml}
|
326
328
|
end
|
329
|
+
|
327
330
|
def generate_output_dir(output_dir)
|
328
331
|
Cnvrg::Logger.log_info("Generating output dir for #{output_dir}")
|
329
332
|
upload_list = []
|
@@ -332,10 +335,10 @@ module Cnvrg
|
|
332
335
|
next if e.end_with? "/."
|
333
336
|
if File.directory? e
|
334
337
|
|
335
|
-
upload_list << e+"/"
|
338
|
+
upload_list << e + "/"
|
336
339
|
else
|
337
340
|
upload_list << e
|
338
|
-
|
341
|
+
end
|
339
342
|
end
|
340
343
|
if Dir.exists? output_dir
|
341
344
|
upload_list << output_dir + "/"
|
@@ -364,130 +367,132 @@ module Cnvrg
|
|
364
367
|
|
365
368
|
end
|
366
369
|
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
370
|
+
def generate_idx(deploy: false)
|
371
|
+
if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
|
372
|
+
old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
373
|
+
else
|
374
|
+
old_idx = {:tree => {}, :commit => nil}
|
375
|
+
end
|
373
376
|
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
377
|
+
tree_idx = Hash.new(0)
|
378
|
+
list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject {|x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~ /^#{self.local_path}\/\.git\/*/) or (x =~ /^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~ /^#{self.local_path}\/\.cnvrgignore/)}
|
379
|
+
list_ignore = self.get_ignore_list()
|
380
|
+
if deploy
|
381
|
+
list_ignore << ["main.py", "main.pyc", "__init__.py", "uwsgi.ini"]
|
382
|
+
list_ignore.flatten!
|
383
|
+
end
|
381
384
|
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
|
385
|
+
Parallel.map(list, in_threads: IDXParallelThreads) do |e|
|
386
|
+
label = e.gsub(self.local_path + "/", "")
|
387
|
+
ignore_label = label.gsub("/", "//")
|
388
|
+
if list_ignore.include? ignore_label
|
389
|
+
next
|
390
|
+
end
|
391
|
+
if File.directory? e
|
389
392
|
|
390
|
-
|
393
|
+
tree_idx[label + "/"] = nil
|
394
|
+
else
|
395
|
+
sha1 = OpenSSL::Digest::SHA1.file(e).hexdigest
|
396
|
+
if old_idx.nil? or old_idx.to_h[:tree].nil?
|
397
|
+
tree_idx[label] = {sha1: sha1, commit_time: nil}
|
398
|
+
elsif old_idx[:tree][label].nil? or old_idx[:tree][label][:sha1] != sha1
|
399
|
+
tree_idx[label] = {sha1: sha1, commit_time: nil}
|
391
400
|
else
|
392
|
-
|
393
|
-
if old_idx.nil? or old_idx.to_h[:tree].nil?
|
394
|
-
tree_idx[label] = {sha1: sha1, commit_time: nil}
|
395
|
-
elsif old_idx[:tree][label].nil? or old_idx[:tree][label][:sha1] != sha1
|
396
|
-
tree_idx[label] = {sha1: sha1, commit_time: nil}
|
397
|
-
else
|
398
|
-
tree_idx[label] = old_idx[:tree][label]
|
399
|
-
end
|
401
|
+
tree_idx[label] = old_idx[:tree][label]
|
400
402
|
end
|
401
403
|
end
|
404
|
+
end
|
402
405
|
|
403
|
-
|
406
|
+
old_idx[:tree] = tree_idx
|
404
407
|
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
408
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write old_idx.to_yaml}
|
409
|
+
return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
410
|
+
end
|
411
|
+
|
412
|
+
def get_idx
|
413
|
+
unless File.exists? "#{self.local_path}/.cnvrg/idx.yml"
|
414
|
+
empty_idx = {:commit => nil, :tree => {}}
|
415
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write empty_idx.to_yaml}
|
416
|
+
return empty_idx
|
415
417
|
end
|
418
|
+
YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
419
|
+
end
|
416
420
|
|
417
421
|
def set_idx(idx)
|
418
422
|
FileUtils.mkdir_p("#{self.local_path}/.cnvrg")
|
419
|
-
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {
|
423
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx.to_yaml}
|
424
|
+
end
|
425
|
+
|
426
|
+
def clone(remote = 0, commit)
|
427
|
+
response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/clone", 'POST', {project_slug: self.slug, remote: remote, commit: commit})
|
428
|
+
return response
|
420
429
|
end
|
421
430
|
|
422
|
-
def clone(remote=0, commit)
|
423
|
-
response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/clone", 'POST', {project_slug: self.slug, remote: remote, commit: commit})
|
424
|
-
return response
|
425
|
-
end
|
426
431
|
def git_download_commit(commit)
|
427
432
|
response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/git_download_commit", 'POST', {commit_sha1: commit})
|
428
|
-
CLI.is_response_success(response,true)
|
433
|
+
CLI.is_response_success(response, true)
|
429
434
|
return response
|
430
435
|
end
|
431
436
|
|
432
437
|
def compare_idx(new_branch, force:false, deploy: false, in_exp:false, specific_files: [], download: false)
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
end
|
451
|
-
response ={"result"=> {"commit"=>nil,"tree"=> {"added"=> added,
|
452
|
-
"updated_on_server"=> [],
|
453
|
-
"updated_on_local"=> [],
|
454
|
-
"update_local" => [],
|
455
|
-
"deleted"=> [],
|
456
|
-
"conflicts"=> []} } }
|
457
|
-
return response
|
438
|
+
is_download = download
|
439
|
+
if is_download
|
440
|
+
local_idx = self.get_idx
|
441
|
+
else
|
442
|
+
#upload
|
443
|
+
local_idx = self.generate_idx(deploy: deploy)
|
444
|
+
end
|
445
|
+
commit = local_idx[:commit]
|
446
|
+
tree = local_idx[:tree]
|
447
|
+
ignore_list = self.send_ignore_list()
|
448
|
+
if force or specific_files.present?
|
449
|
+
added = []
|
450
|
+
if specific_files.present?
|
451
|
+
added = specific_files
|
452
|
+
elsif tree.present?
|
453
|
+
added << local_idx[:tree].keys
|
454
|
+
added.flatten!
|
458
455
|
end
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
456
|
+
response = {"result" => {"commit" => nil, "tree" => {"added" => added,
|
457
|
+
"updated_on_server" => [],
|
458
|
+
"updated_on_local" => [],
|
459
|
+
"update_local" => [],
|
460
|
+
"deleted" => [],
|
461
|
+
"conflicts" => []}}}
|
462
|
+
return response
|
463
|
+
end
|
464
|
+
#we dont want to send it on download - we only compare between commits sha1 in download.
|
465
|
+
if is_download
|
466
|
+
#the new server doesnt need the tree, but the old probably needs :X
|
467
|
+
local_idx[:tree] = {} if Cnvrg::Helpers.server_version > 0
|
468
|
+
end
|
469
|
+
response = Cnvrg::API.request(@base_resource + "status", 'POST', {idx: local_idx, new_branch: new_branch,
|
470
|
+
current_commit: commit, ignore: ignore_list, force: force, in_exp: in_exp, download: download})
|
471
|
+
|
472
|
+
CLI.is_response_success(response, true)
|
473
|
+
if is_download
|
474
|
+
if Cnvrg::Helpers.server_version > 0
|
475
|
+
#trying to optimize the download using resolver
|
476
|
+
resolve = response['result']['tree']['resolver'] || tree #tree of file -> sha1 from current commit to check conflicts
|
477
|
+
destination_files = response['result']['tree']['destination'] || {} #tree of file -> sha1 from last commit to check files that already downloaded
|
478
|
+
else
|
479
|
+
resolve = tree
|
480
|
+
destination_files = {}
|
463
481
|
end
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
CLI.is_response_success(response,true)
|
468
|
-
if is_download
|
469
|
-
if Cnvrg::Helpers.server_version > 0
|
470
|
-
#trying to optimize the download using resolver
|
471
|
-
resolve = response['result']['tree']['resolver'] || tree #tree of file -> sha1 from current commit to check conflicts
|
472
|
-
destination_files = response['result']['tree']['destination'] || {} #tree of file -> sha1 from last commit to check files that already downloaded
|
473
|
-
else
|
474
|
-
resolve = tree
|
475
|
-
destination_files = {}
|
476
|
-
end
|
477
|
-
@files = self.get_files
|
478
|
-
local_tree = @files.calculate_sha1(resolve.keys)
|
479
|
-
changed_files = resolve.keys.select{|file| local_tree[file] != resolve[file]}
|
482
|
+
@files = self.get_files
|
483
|
+
local_tree = @files.calculate_sha1(resolve.keys)
|
484
|
+
changed_files = resolve.keys.select {|file| local_tree[file] != resolve[file]}
|
480
485
|
|
481
|
-
|
482
|
-
|
486
|
+
# means that the user changed the file locally
|
487
|
+
response['result']['tree']['update_local'] = changed_files
|
483
488
|
|
484
|
-
|
485
|
-
|
486
|
-
|
487
|
-
|
488
|
-
end
|
489
|
-
return response
|
489
|
+
# means that we already downloaded this file and we dont need it anymore
|
490
|
+
downloaded_files = destination_files.keys.select {|file| local_tree[file] == destination_files[file]}
|
491
|
+
response['result']['tree']['added'] -= downloaded_files
|
492
|
+
response['result']['tree']['updated_on_server'] -= downloaded_files
|
490
493
|
end
|
494
|
+
return response
|
495
|
+
end
|
491
496
|
|
492
497
|
def get_files
|
493
498
|
Cnvrg::Files.new(self.owner, self.slug, project_home: @local_path)
|
@@ -502,52 +507,52 @@ module Cnvrg
|
|
502
507
|
'POST',
|
503
508
|
{tree: tree, ignore: ignore_list,
|
504
509
|
dest_commit: destination, current_commit: current_commit})
|
505
|
-
CLI.is_response_success(response,false)
|
510
|
+
CLI.is_response_success(response, false)
|
506
511
|
response
|
507
512
|
end
|
508
513
|
|
509
514
|
def set_on_branch(is_latest)
|
510
|
-
config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
|
515
|
+
config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
|
511
516
|
@is_branch = !is_latest
|
512
517
|
config[:is_branch] = @is_branch
|
513
518
|
|
514
|
-
File.open(@working_dir+"/.cnvrg/config.yml", 'w') {
|
519
|
+
File.open(@working_dir + "/.cnvrg/config.yml", 'w') {|f| f.write config.to_yaml}
|
515
520
|
|
516
521
|
end
|
517
522
|
|
518
|
-
|
519
|
-
|
520
|
-
|
521
|
-
end
|
522
|
-
response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
|
523
|
-
CLI.is_response_success(response,false)
|
524
|
-
update_is_new_branch(response["result"]["new_branch"])
|
525
|
-
return response["result"]["new_branch"]
|
523
|
+
def compare_commit(commit)
|
524
|
+
if commit.nil? or commit.empty?
|
525
|
+
commit = last_local_commit
|
526
526
|
end
|
527
|
+
response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
|
528
|
+
CLI.is_response_success(response, false)
|
529
|
+
update_is_new_branch(response["result"]["new_branch"])
|
530
|
+
return response["result"]["new_branch"]
|
531
|
+
end
|
527
532
|
|
528
|
-
|
529
|
-
|
530
|
-
idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
531
|
-
files.each do |path|
|
532
|
-
idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
|
533
|
-
end
|
534
|
-
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
|
533
|
+
def update_idx_with_files_commits!(files, commit_time)
|
535
534
|
|
536
|
-
|
535
|
+
idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
536
|
+
files.each do |path|
|
537
|
+
idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
|
537
538
|
end
|
539
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
|
540
|
+
|
541
|
+
return true
|
542
|
+
end
|
538
543
|
|
539
|
-
|
544
|
+
def deploy(file_to_run, function, input_params, commit_to_run, instance_type, image_slug, scheduling_query, local_timestamp, workers, file_input, title)
|
545
|
+
response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/deploy", 'POST', {file_to_run: file_to_run, function: function,
|
546
|
+
image_slug: image_slug, input_params: input_params,
|
547
|
+
commit_sha1: commit_to_run,
|
548
|
+
instance_type: instance_type,
|
549
|
+
scheduling_query: scheduling_query,
|
550
|
+
local_timestamp: local_timestamp,
|
551
|
+
workers: workers, file_input: file_input,
|
552
|
+
title: title})
|
553
|
+
return response
|
554
|
+
end
|
540
555
|
|
541
|
-
response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/deploy", 'POST', {file_to_run: file_to_run, function: function,
|
542
|
-
image_slug: image_slug, input_params: input_params,
|
543
|
-
commit_sha1: commit_to_run,
|
544
|
-
instance_type: instance_type,
|
545
|
-
scheduling_query: scheduling_query,
|
546
|
-
local_timestamp: local_timestamp,
|
547
|
-
workers:workers,file_input:file_input,
|
548
|
-
title: title})
|
549
|
-
return response
|
550
|
-
end
|
551
556
|
def list_commits
|
552
557
|
response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commits/list", 'GET')
|
553
558
|
CLI.is_response_success(response)
|
@@ -567,17 +572,19 @@ module Cnvrg
|
|
567
572
|
CLI.is_response_success(response)
|
568
573
|
return response
|
569
574
|
end
|
570
|
-
def update_idx_with_commit!(commit)
|
571
|
-
idx_hash = self.get_idx
|
572
|
-
idx_hash[:commit] = commit
|
573
575
|
|
574
|
-
|
575
|
-
|
576
|
-
|
576
|
+
def update_idx_with_commit!(commit, latest: nil)
|
577
|
+
idx_hash = self.get_idx
|
578
|
+
idx_hash[:commit] = commit
|
579
|
+
self.set_on_branch(latest) unless latest.nil?
|
577
580
|
|
578
|
-
|
579
|
-
|
580
|
-
|
581
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
|
582
|
+
return true
|
583
|
+
end
|
584
|
+
|
585
|
+
def revert(working_dir)
|
586
|
+
FileUtils.rm_rf working_dir
|
587
|
+
end
|
581
588
|
|
582
589
|
def init_machines
|
583
590
|
Cnvrg::Logger.log_info("Init machines.")
|
@@ -599,5 +606,6 @@ module Cnvrg
|
|
599
606
|
machines.include? machine
|
600
607
|
end
|
601
608
|
|
602
|
-
|
609
|
+
|
603
610
|
end
|
611
|
+
end
|
data/lib/cnvrg/version.rb
CHANGED