cnvrg 0.8.5 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +54 -192
- data/lib/cnvrg/files.rb +26 -6
- data/lib/cnvrg/project.rb +265 -257
- data/lib/cnvrg/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 819730717280428ac1ba231d43462cb827253cc1
|
|
4
|
+
data.tar.gz: 95c1963b178296af0655d54b691b02895ced41ef
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 61563035b65f94b7d87354a6dcb783119179edf655cd6881595035417d15c6441b0337f48ae9de9b410136bd3133eb3254fb995af46ffbd67d785e09312aa3e2
|
|
7
|
+
data.tar.gz: fda3bfcd38199e6c04c46a2c2ca7812182e58352c0a0c9d06898acca3f5de30178160d35ab6b6d86d640d7c89ecb1a773a562d44af369e652b5e476943aa8c21
|
data/lib/cnvrg/cli.rb
CHANGED
|
@@ -305,7 +305,7 @@ module Cnvrg
|
|
|
305
305
|
compression_path = "#{home_dir}/.cnvrg/tmp"
|
|
306
306
|
config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url, compression_path: compression_path}
|
|
307
307
|
File.open(home_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
|
|
308
|
-
say "Done"
|
|
308
|
+
say "Done", Thor::Shell::Color::GREEN
|
|
309
309
|
rescue
|
|
310
310
|
say "ERROR", Thor::Shell::Color::RED
|
|
311
311
|
File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
|
@@ -340,7 +340,7 @@ module Cnvrg
|
|
|
340
340
|
compression_path = "#{home_dir}/.cnvrg/tmp"
|
|
341
341
|
config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url, compression_path: compression_path}
|
|
342
342
|
File.open(home_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
|
|
343
|
-
say "Done"
|
|
343
|
+
say "Done", Thor::Shell::Color::GREEN
|
|
344
344
|
rescue
|
|
345
345
|
say "ERROR", Thor::Shell::Color::RED
|
|
346
346
|
end
|
|
@@ -408,7 +408,7 @@ module Cnvrg
|
|
|
408
408
|
end
|
|
409
409
|
end
|
|
410
410
|
rescue SignalException
|
|
411
|
-
say "\nAborting"
|
|
411
|
+
say "\nAborting", Thor::Shell::Color::RED
|
|
412
412
|
exit(1)
|
|
413
413
|
end
|
|
414
414
|
end
|
|
@@ -442,7 +442,7 @@ module Cnvrg
|
|
|
442
442
|
say "#{checks} Done", Thor::Shell::Color::GREEN
|
|
443
443
|
|
|
444
444
|
rescue SignalException
|
|
445
|
-
say "\nAborting"
|
|
445
|
+
say "\nAborting", Thor::Shell::Color::RED
|
|
446
446
|
exit(1)
|
|
447
447
|
end
|
|
448
448
|
end
|
|
@@ -544,7 +544,7 @@ module Cnvrg
|
|
|
544
544
|
exit(1)
|
|
545
545
|
rescue SignalException
|
|
546
546
|
|
|
547
|
-
say "\nAborting"
|
|
547
|
+
say "\nAborting",Thor::Shell::Color::RED
|
|
548
548
|
logout()
|
|
549
549
|
exit(1)
|
|
550
550
|
end
|
|
@@ -565,7 +565,7 @@ module Cnvrg
|
|
|
565
565
|
puts e.message
|
|
566
566
|
puts e.backtrace
|
|
567
567
|
rescue SignalException
|
|
568
|
-
say "\nAborting"
|
|
568
|
+
say "\nAborting",Thor::Shell::Color::RED
|
|
569
569
|
exit(1)
|
|
570
570
|
end
|
|
571
571
|
|
|
@@ -587,7 +587,7 @@ module Cnvrg
|
|
|
587
587
|
|
|
588
588
|
|
|
589
589
|
rescue SignalException
|
|
590
|
-
say "\nAborting"
|
|
590
|
+
say "\nAborting", Thor::Shell::Color::RED
|
|
591
591
|
exit(1)
|
|
592
592
|
end
|
|
593
593
|
end
|
|
@@ -676,7 +676,7 @@ module Cnvrg
|
|
|
676
676
|
|
|
677
677
|
@project.revert(working_dir)
|
|
678
678
|
end
|
|
679
|
-
say "\nAborting"
|
|
679
|
+
say "\nAborting", Thor::Shell::Color::RED
|
|
680
680
|
exit(1)
|
|
681
681
|
end
|
|
682
682
|
end
|
|
@@ -762,7 +762,7 @@ module Cnvrg
|
|
|
762
762
|
log_error(e)
|
|
763
763
|
rescue SignalException
|
|
764
764
|
|
|
765
|
-
say "\nAborting"
|
|
765
|
+
say "\nAborting", Thor::Shell::Color::RED
|
|
766
766
|
exit(1)
|
|
767
767
|
end
|
|
768
768
|
end
|
|
@@ -802,7 +802,7 @@ module Cnvrg
|
|
|
802
802
|
rescue => e
|
|
803
803
|
log_error(e)
|
|
804
804
|
rescue SignalException
|
|
805
|
-
say "\nAborting"
|
|
805
|
+
say "\nAborting", Thor::Shell::Color::RED
|
|
806
806
|
exit(1)
|
|
807
807
|
end
|
|
808
808
|
end
|
|
@@ -826,7 +826,7 @@ module Cnvrg
|
|
|
826
826
|
log_error(e)
|
|
827
827
|
rescue SignalException
|
|
828
828
|
|
|
829
|
-
say "\nAborting"
|
|
829
|
+
say "\nAborting", Thor::Shell::Color::RED
|
|
830
830
|
exit(1)
|
|
831
831
|
end
|
|
832
832
|
end
|
|
@@ -891,6 +891,7 @@ module Cnvrg
|
|
|
891
891
|
downloaded_files += files['keys'].length
|
|
892
892
|
files = @files.get_clone_chunk(commit: commit, latest_id: files['latest'])
|
|
893
893
|
end
|
|
894
|
+
progressbar.finish
|
|
894
895
|
if downloaded_files == files_count
|
|
895
896
|
Dataset.verify_cnvrgignore_exist(dataset_name, false)
|
|
896
897
|
@dataset = Dataset.new(dataset_home)
|
|
@@ -905,7 +906,7 @@ module Cnvrg
|
|
|
905
906
|
exit(1)
|
|
906
907
|
end
|
|
907
908
|
rescue SignalException
|
|
908
|
-
say "\nAborting"
|
|
909
|
+
say "\nAborting", Thor::Shell::Color::RED
|
|
909
910
|
exit(1)
|
|
910
911
|
end
|
|
911
912
|
end
|
|
@@ -983,7 +984,7 @@ module Cnvrg
|
|
|
983
984
|
end
|
|
984
985
|
end
|
|
985
986
|
rescue SignalException
|
|
986
|
-
say "\nAborting"
|
|
987
|
+
say "\nAborting", Thor::Shell::Color::RED
|
|
987
988
|
exit(1)
|
|
988
989
|
end
|
|
989
990
|
end
|
|
@@ -1009,7 +1010,7 @@ module Cnvrg
|
|
|
1009
1010
|
|
|
1010
1011
|
rescue SignalException
|
|
1011
1012
|
|
|
1012
|
-
say "\nAborting"
|
|
1013
|
+
say "\nAborting", Thor::Shell::Color::RED
|
|
1013
1014
|
exit(1)
|
|
1014
1015
|
end
|
|
1015
1016
|
end
|
|
@@ -1031,7 +1032,7 @@ module Cnvrg
|
|
|
1031
1032
|
rescue SignalException
|
|
1032
1033
|
log_end(-1)
|
|
1033
1034
|
|
|
1034
|
-
say "\nAborting"
|
|
1035
|
+
say "\nAborting", Thor::Shell::Color::RED
|
|
1035
1036
|
exit(1)
|
|
1036
1037
|
end
|
|
1037
1038
|
end
|
|
@@ -1813,112 +1814,22 @@ module Cnvrg
|
|
|
1813
1814
|
|
|
1814
1815
|
if clone_resp
|
|
1815
1816
|
@project = Project.new(project_home)
|
|
1816
|
-
@files = Cnvrg::Files.new(@project.owner, slug)
|
|
1817
|
+
@files = Cnvrg::Files.new(@project.owner, slug, project_home: project_home)
|
|
1817
1818
|
response = @project.clone(remote, commit_to_clone)
|
|
1818
1819
|
Cnvrg::CLI.is_response_success response
|
|
1820
|
+
commit_sha1 = response["result"]["commit"]
|
|
1821
|
+
files = response["result"]["tree"].keys
|
|
1819
1822
|
idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
|
|
1820
|
-
|
|
1821
1823
|
log_message("Downloading files", Thor::Shell::Color::BLUE)
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
rescue Interrupt
|
|
1830
|
-
log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
|
|
1831
|
-
|
|
1832
|
-
@files.revoke_clone(project_home)
|
|
1833
|
-
return
|
|
1834
|
-
end
|
|
1835
|
-
Project.verify_cnvrgignore_exist(project_name, remote)
|
|
1836
|
-
if !download_result.is_success?
|
|
1837
|
-
log_message(download_result.e_msg, download_result.msg_color, false)
|
|
1838
|
-
log_message(download_result.e_msg_backtrace, download_result.msg_color, false)
|
|
1839
|
-
|
|
1840
|
-
@files.revoke_clone(project_home)
|
|
1841
|
-
end
|
|
1842
|
-
log_message(download_result.msg, download_result.msg_color)
|
|
1843
|
-
else
|
|
1844
|
-
current_commit = response["result"]["commit"]
|
|
1845
|
-
if !response["result"]["tree"].nil?
|
|
1846
|
-
parallel_options = {
|
|
1847
|
-
:progress => {
|
|
1848
|
-
:title => "Download Progress",
|
|
1849
|
-
:progress_mark => '=',
|
|
1850
|
-
:format => "%b>>%i| %p%% %t",
|
|
1851
|
-
:starting_at => 0,
|
|
1852
|
-
:total => response["result"]["tree"].size,
|
|
1853
|
-
:autofinish => true
|
|
1854
|
-
},
|
|
1855
|
-
in_processes: ParallelProcesses,
|
|
1856
|
-
in_thread: ParallelThreads
|
|
1857
|
-
}
|
|
1858
|
-
begin
|
|
1859
|
-
successful_changes =[]
|
|
1860
|
-
is_success = true
|
|
1861
|
-
clone_result = Parallel.map((response["result"]["tree"]), parallel_options) do |f|
|
|
1862
|
-
|
|
1863
|
-
relative_path = f[0].gsub(/^#{@project.local_path}/, "")
|
|
1864
|
-
if f[0].end_with? "/"
|
|
1865
|
-
# dir
|
|
1866
|
-
if @files.download_dir(f[0], relative_path, project_home)
|
|
1867
|
-
f
|
|
1868
|
-
successful_changes << relative_path
|
|
1869
|
-
else
|
|
1870
|
-
is_success =false
|
|
1871
|
-
log_message("Could not create directory: #{f[0]}", Thor::Shell::Color::RED)
|
|
1872
|
-
raise Parallel::Kill
|
|
1873
|
-
end
|
|
1874
|
-
else
|
|
1875
|
-
# blob
|
|
1876
|
-
|
|
1877
|
-
if @files.download_file_s3(f[0], relative_path, project_home, commit_sha1=current_commit)
|
|
1878
|
-
f
|
|
1879
|
-
successful_changes << relative_path
|
|
1880
|
-
else
|
|
1881
|
-
is_success =false
|
|
1882
|
-
log_message("Could not download file: #{f[0]}", Thor::Shell::Color::RED)
|
|
1883
|
-
raise Parallel::Kill
|
|
1884
|
-
|
|
1885
|
-
end
|
|
1886
|
-
end
|
|
1887
|
-
end
|
|
1888
|
-
successful_changes = response["result"]["tree"]
|
|
1889
|
-
if !successful_changes.nil? and is_success
|
|
1890
|
-
@project.set_idx(idx)
|
|
1891
|
-
Project.verify_cnvrgignore_exist(project_name,remote)
|
|
1892
|
-
log_message("Done.\nDownloaded #{successful_changes.size}/#{response["result"]["tree"].size} files", Thor::Shell::Color::GREEN)
|
|
1893
|
-
else
|
|
1894
|
-
log_message("Couldn't download some files", Thor::Shell::Color::RED)
|
|
1895
|
-
|
|
1896
|
-
end
|
|
1897
|
-
rescue Interrupt
|
|
1898
|
-
log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
|
|
1899
|
-
|
|
1900
|
-
@files.revoke_clone(project_home)
|
|
1901
|
-
return
|
|
1902
|
-
end
|
|
1903
|
-
end
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
end
|
|
1907
|
-
|
|
1908
|
-
else
|
|
1909
|
-
|
|
1910
|
-
log_message("Error: Couldn't clone: #{project_name}", Thor::Shell::Color::RED)
|
|
1911
|
-
@files.revoke_clone(project_home)
|
|
1912
|
-
|
|
1913
|
-
return
|
|
1824
|
+
progressbar = @files.create_progressbar(files.size, "Clone Progress")
|
|
1825
|
+
@files.download_files(files, commit_sha1, progress: progressbar)
|
|
1826
|
+
progressbar.finish
|
|
1827
|
+
Project.verify_cnvrgignore_exist(project_name, remote)
|
|
1828
|
+
@project.set_idx(idx)
|
|
1829
|
+
log_message("Done")
|
|
1830
|
+
log_message("Downloaded #{files.size} files")
|
|
1914
1831
|
end
|
|
1915
|
-
rescue SignalException
|
|
1916
|
-
say "\nAborting"
|
|
1917
|
-
@files.revoke_clone(project_home)
|
|
1918
|
-
|
|
1919
|
-
return
|
|
1920
1832
|
end
|
|
1921
|
-
|
|
1922
1833
|
end
|
|
1923
1834
|
|
|
1924
1835
|
|
|
@@ -2360,7 +2271,7 @@ module Cnvrg
|
|
|
2360
2271
|
:autofinish => true)
|
|
2361
2272
|
@files.upload_multiple_files(to_upload, commit_sha1, progress: progressbar)
|
|
2362
2273
|
@files.delete_files_from_server(deleted, commit_sha1)
|
|
2363
|
-
progressbar.
|
|
2274
|
+
progressbar.finish
|
|
2364
2275
|
res = @files.end_commit(commit_sha1, force: force, message: commit_msg)
|
|
2365
2276
|
unless Cnvrg::CLI.is_response_success(res, false)
|
|
2366
2277
|
raise StandardError.new("Cant end commit")
|
|
@@ -2614,6 +2525,9 @@ module Cnvrg
|
|
|
2614
2525
|
if git or @project.is_git
|
|
2615
2526
|
return download_in_git(commit)
|
|
2616
2527
|
end
|
|
2528
|
+
if commit.present?
|
|
2529
|
+
return jump(commit)
|
|
2530
|
+
end
|
|
2617
2531
|
ignore = options[:ignore] || ""
|
|
2618
2532
|
if ignore.nil? or ignore.empty?
|
|
2619
2533
|
ignore = ignore_list
|
|
@@ -2683,17 +2597,20 @@ module Cnvrg
|
|
|
2683
2597
|
|
|
2684
2598
|
|
|
2685
2599
|
Cnvrg::Logger.log_info("Downloading updated files:#{updated_files.join(",")}")
|
|
2686
|
-
@files.download_files(updated_files, commit, progress: progressbar)
|
|
2600
|
+
@files.download_files(updated_files, commit, progress: progressbar)
|
|
2687
2601
|
|
|
2688
2602
|
Cnvrg::Logger.log_info("Downloading conflicted files:#{conflicted_files.join(",")}")
|
|
2689
|
-
@files.download_files(conflicted_files, commit, postfix: ".conflict", progress: progressbar)
|
|
2603
|
+
@files.download_files(conflicted_files, commit, postfix: ".conflict", progress: progressbar)
|
|
2690
2604
|
|
|
2691
2605
|
Cnvrg::Logger.log_info("Delete files: #{deleted_files.join(",")}")
|
|
2692
2606
|
@files.delete_files_local(deleted_files, conflicted: conflicted_deleted, progress: progressbar)
|
|
2693
2607
|
|
|
2694
2608
|
# update idx with latest commit
|
|
2695
|
-
|
|
2609
|
+
# the latest true its because if we define --commit in the cmd it will go to "def jump(options['commit'])"
|
|
2610
|
+
# so if we are downloads something, we have to stay here.
|
|
2611
|
+
@project.update_idx_with_commit!(commit, latest: true)
|
|
2696
2612
|
#TODO Sync, remove idx, sync again and pray
|
|
2613
|
+
progressbar.finish
|
|
2697
2614
|
check = Helpers.checkmark()
|
|
2698
2615
|
Cnvrg::Logger.log_info("Finished downloading successfuly")
|
|
2699
2616
|
if options["verbose"]
|
|
@@ -2728,10 +2645,11 @@ module Cnvrg
|
|
|
2728
2645
|
project_home = get_project_home
|
|
2729
2646
|
@project = Project.new(project_home)
|
|
2730
2647
|
current_commit = @project.last_local_commit
|
|
2731
|
-
if current_commit.
|
|
2648
|
+
if current_commit.start_with? commit_sha1 #commit_sha1 can be partial.
|
|
2732
2649
|
log_message("Project is already updated", Thor::Shell::Color::GREEN)
|
|
2733
2650
|
exit(0)
|
|
2734
2651
|
end
|
|
2652
|
+
log_message("Jumping to commit #{commit_sha1}")
|
|
2735
2653
|
@files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: project_home)
|
|
2736
2654
|
resp = @project.jump_idx(destination: commit_sha1)
|
|
2737
2655
|
if resp.blank?
|
|
@@ -2739,81 +2657,27 @@ module Cnvrg
|
|
|
2739
2657
|
exit(0)
|
|
2740
2658
|
end
|
|
2741
2659
|
compare = resp['result']['compare']
|
|
2742
|
-
resolver = resp['result']['resolver']
|
|
2743
2660
|
latest = resp['result']['latest']
|
|
2744
2661
|
commit = resp['result']['commit']
|
|
2745
|
-
updated_files = compare['updated_on_server']
|
|
2662
|
+
updated_files = compare['updated_on_server'] + compare['added']
|
|
2746
2663
|
conflicted_files = compare['conflicts']
|
|
2664
|
+
conflicted_deleted = compare['delete_conflicts'] || []
|
|
2747
2665
|
deleted_files = compare['deleted']
|
|
2748
2666
|
overall_changes = [updated_files, conflicted_files, deleted_files].flatten.size
|
|
2749
2667
|
|
|
2750
|
-
|
|
2751
|
-
progressbar = ProgressBar.create(:title => "Download Progress",
|
|
2752
|
-
:progress_mark => '=',
|
|
2753
|
-
:format => "%b>>%i| %p%% %t",
|
|
2754
|
-
:starting_at => 0,
|
|
2755
|
-
:total => overall_changes,
|
|
2756
|
-
:autofinish => true)
|
|
2757
|
-
|
|
2668
|
+
progressbar = @files.create_progressbar(overall_changes, "Download Progress")
|
|
2758
2669
|
@files.download_files(updated_files, commit_sha1, progress: progressbar)
|
|
2759
2670
|
@files.download_files(conflicted_files, commit_sha1, progress: progressbar, postfix: '.conflicted')
|
|
2760
|
-
@files.delete_files_local(deleted_files, progress: progressbar)
|
|
2761
|
-
|
|
2762
|
-
|
|
2763
|
-
@project.set_on_branch(latest)
|
|
2764
|
-
@project.update_idx_with_commit!(commit)
|
|
2671
|
+
@files.delete_files_local(deleted_files, progress: progressbar, conflicted: conflicted_deleted)
|
|
2672
|
+
progressbar.finish
|
|
2673
|
+
@project.update_idx_with_commit!(commit, latest: latest)
|
|
2765
2674
|
@project.generate_idx
|
|
2675
|
+
|
|
2766
2676
|
log_message("Jumped successfuly!", Thor::Shell::Color::GREEN)
|
|
2767
2677
|
rescue => e
|
|
2678
|
+
Logger::log_error(e)
|
|
2768
2679
|
log_message("Cant jump to the specified commit", Thor::Shell::Color::RED)
|
|
2769
2680
|
end
|
|
2770
|
-
# successful_changes = []
|
|
2771
|
-
#
|
|
2772
|
-
# if !response["result"]["tree"].nil?
|
|
2773
|
-
|
|
2774
|
-
# commit_sha1 = response["result"]["commit"]
|
|
2775
|
-
# idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
|
|
2776
|
-
# File.open(project_home + "/.cnvrg/idx.yml", "w+") {|f| f.write idx.to_yaml}
|
|
2777
|
-
# if is_remote
|
|
2778
|
-
# current_tree = Dir.glob("**/*", File::FNM_DOTMATCH).flatten.reject {|file| file.start_with? '.' or file.eql? "__init__.py" or file.eql? "uwsgi.ini" or file.ends_with? "/." or file.eql? "."}
|
|
2779
|
-
# else
|
|
2780
|
-
# current_tree = Dir.glob("**/*", File::FNM_DOTMATCH).flatten.reject {|file| file.start_with? '.cnvrg' or file.ends_with? "/." or file.eql? "."}
|
|
2781
|
-
# end
|
|
2782
|
-
#
|
|
2783
|
-
# jump_result = Parallel.map(response["result"]["tree"], parallel_options) do |f|
|
|
2784
|
-
#
|
|
2785
|
-
# relative_path = f[0].gsub(/^#{@project.local_path}/, "")
|
|
2786
|
-
# log_message("Downloading #{f[0]}", Thor::Shell::Color::BLUE, false)
|
|
2787
|
-
# if f[0].end_with? "/"
|
|
2788
|
-
# # dir
|
|
2789
|
-
# @files.download_dir(f[0], relative_path, project_home)
|
|
2790
|
-
#
|
|
2791
|
-
# else
|
|
2792
|
-
# # blob
|
|
2793
|
-
# @files.download_file_s3(f[0], relative_path, project_home, commit_sha1 = commit_sha1)
|
|
2794
|
-
# end
|
|
2795
|
-
# end
|
|
2796
|
-
#
|
|
2797
|
-
#
|
|
2798
|
-
# successful_changes = jump_result.select {|x| not x.nil?}
|
|
2799
|
-
# end
|
|
2800
|
-
# response["result"]["tree"].each do |f|
|
|
2801
|
-
# if f[0].end_with? "/"
|
|
2802
|
-
# current_tree.delete(f[0][0, f[0].size - 1])
|
|
2803
|
-
# else
|
|
2804
|
-
# current_tree.delete(f[0])
|
|
2805
|
-
# end
|
|
2806
|
-
# end
|
|
2807
|
-
#
|
|
2808
|
-
# FileUtils.rm_rf(current_tree)
|
|
2809
|
-
# log_message("Done. Jumped to #{commit_sha1} completed successfully", Thor::Shell::Color::GREEN)
|
|
2810
|
-
# rescue => e
|
|
2811
|
-
# log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
|
|
2812
|
-
# log_error(e)
|
|
2813
|
-
# rescue SignalException
|
|
2814
|
-
#
|
|
2815
|
-
# exit(1)
|
|
2816
|
-
# end
|
|
2817
2681
|
end
|
|
2818
2682
|
|
|
2819
2683
|
desc 'show', 'Show specific file from a specific commit'
|
|
@@ -2911,6 +2775,7 @@ module Cnvrg
|
|
|
2911
2775
|
method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
|
|
2912
2776
|
def sync(direct = true)
|
|
2913
2777
|
verify_logged_in(true) if direct
|
|
2778
|
+
@project = Project.new(get_project_home)
|
|
2914
2779
|
log_start(__method__, args, options)
|
|
2915
2780
|
log_message('Checking for new updates from remote version', Thor::Shell::Color::BLUE, options["verbose"])
|
|
2916
2781
|
log_message('Syncing project', Thor::Shell::Color::BLUE, !options["verbose"])
|
|
@@ -2919,7 +2784,7 @@ module Cnvrg
|
|
|
2919
2784
|
in_exp = options["in_exp"] || (job_slug.present? and job_type.present?)
|
|
2920
2785
|
in_exp = false if job_type.present? and job_type == "NotebookSession"
|
|
2921
2786
|
run_download = true
|
|
2922
|
-
if options[:force] or options[:files].present? or options[:output_dir].present? or in_exp
|
|
2787
|
+
if options[:force] or options[:files].present? or options[:output_dir].present? or in_exp or @project.is_branch
|
|
2923
2788
|
run_download = false
|
|
2924
2789
|
end
|
|
2925
2790
|
if run_download
|
|
@@ -3477,10 +3342,6 @@ module Cnvrg
|
|
|
3477
3342
|
else
|
|
3478
3343
|
sync_result = invoke :sync, [false], :force => false
|
|
3479
3344
|
end
|
|
3480
|
-
|
|
3481
|
-
|
|
3482
|
-
|
|
3483
|
-
|
|
3484
3345
|
end
|
|
3485
3346
|
#handle grid if it's git project
|
|
3486
3347
|
if project.is_git and grid.present?
|
|
@@ -3516,6 +3377,7 @@ module Cnvrg
|
|
|
3516
3377
|
if forced_commit and (commit_to_run.nil? or commit_to_run.empty?)
|
|
3517
3378
|
commit_to_run = forced_commit
|
|
3518
3379
|
end
|
|
3380
|
+
|
|
3519
3381
|
commit_to_run = commit_to_run.presence || project.last_local_commit
|
|
3520
3382
|
|
|
3521
3383
|
res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
|
|
@@ -5491,7 +5353,7 @@ module Cnvrg
|
|
|
5491
5353
|
end
|
|
5492
5354
|
end
|
|
5493
5355
|
|
|
5494
|
-
def log_message(message, type, to_print = true)
|
|
5356
|
+
def log_message(message, type=Thor::Shell::Color::GREEN, to_print = true)
|
|
5495
5357
|
if to_print
|
|
5496
5358
|
say message, type
|
|
5497
5359
|
end
|
|
@@ -5538,10 +5400,10 @@ module Cnvrg
|
|
|
5538
5400
|
error = response['message'] || "Unknown error"
|
|
5539
5401
|
# Cnvrg::CLI.log_end(1, error)
|
|
5540
5402
|
if response["status"] == 500
|
|
5541
|
-
|
|
5403
|
+
log_message('Server Error', Thor::Shell::Color::RED)
|
|
5542
5404
|
else
|
|
5543
5405
|
$LOG.error message: error, type: "error"
|
|
5544
|
-
|
|
5406
|
+
log_message("error", Thor::Shell::Color::RED)
|
|
5545
5407
|
end
|
|
5546
5408
|
|
|
5547
5409
|
if should_exit
|
|
@@ -5555,7 +5417,7 @@ module Cnvrg
|
|
|
5555
5417
|
puts e.message
|
|
5556
5418
|
puts e.backtrace
|
|
5557
5419
|
rescue SignalException
|
|
5558
|
-
|
|
5420
|
+
log_message "Aborting", Thor::Shell::Color::RED
|
|
5559
5421
|
end
|
|
5560
5422
|
|
|
5561
5423
|
end
|
|
@@ -5837,7 +5699,7 @@ module Cnvrg
|
|
|
5837
5699
|
rescue
|
|
5838
5700
|
return false
|
|
5839
5701
|
rescue SignalException
|
|
5840
|
-
say "\nAborting"
|
|
5702
|
+
say "\nAborting", Thor::Shell::Color::RED
|
|
5841
5703
|
Exit(0)
|
|
5842
5704
|
end
|
|
5843
5705
|
|
data/lib/cnvrg/files.rb
CHANGED
|
@@ -567,17 +567,15 @@ module Cnvrg
|
|
|
567
567
|
end
|
|
568
568
|
|
|
569
569
|
|
|
570
|
-
def download_file_s3(
|
|
570
|
+
def download_file_s3(relative_path, commit_sha1=nil, postfix: '')
|
|
571
571
|
begin
|
|
572
|
-
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {
|
|
572
|
+
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {relative_path: relative_path,
|
|
573
573
|
commit_sha1: commit_sha1,new_version:true})
|
|
574
574
|
|
|
575
575
|
Cnvrg::CLI.is_response_success(res, false)
|
|
576
576
|
if res["result"]
|
|
577
577
|
download_resp = res
|
|
578
578
|
filename = download_resp["result"]["filename"]
|
|
579
|
-
|
|
580
|
-
absolute_path += ".conflict" if conflict
|
|
581
579
|
sts_path = download_resp["result"]["path_sts"]
|
|
582
580
|
retries = 0
|
|
583
581
|
success= false
|
|
@@ -632,8 +630,8 @@ module Cnvrg
|
|
|
632
630
|
:endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
|
|
633
631
|
:http_open_timeout => 60, :retry_limit => 20)
|
|
634
632
|
end
|
|
635
|
-
|
|
636
|
-
File.open(
|
|
633
|
+
absolute_path = "#{@project_home}/#{relative_path}#{postfix}"
|
|
634
|
+
File.open(absolute_path, 'w+') do |file|
|
|
637
635
|
resp = client.get_object({bucket:bucket,
|
|
638
636
|
key:file_key}, target: file)
|
|
639
637
|
end
|
|
@@ -649,7 +647,21 @@ module Cnvrg
|
|
|
649
647
|
end
|
|
650
648
|
end
|
|
651
649
|
|
|
650
|
+
def create_progressbar(length = 10, title = 'Progress')
|
|
651
|
+
ProgressBar.create(:title => title,
|
|
652
|
+
:progress_mark => '=',
|
|
653
|
+
:format => "%b>>%i| %p%% %t",
|
|
654
|
+
:starting_at => 0,
|
|
655
|
+
:total => length,
|
|
656
|
+
:autofinish => true)
|
|
657
|
+
end
|
|
658
|
+
|
|
652
659
|
def download_files(files, commit, postfix: '', progress: nil)
|
|
660
|
+
return if files.blank?
|
|
661
|
+
if Cnvrg::Helpers.server_version < 1
|
|
662
|
+
Cnvrg::Logger.log_info("Download files from older server.")
|
|
663
|
+
return self.download_files_old(files, commit, progress: progress, postfix: postfix)
|
|
664
|
+
end
|
|
653
665
|
res = Cnvrg::API.request(@base_resource + "download_files", 'POST', {files: files, commit: commit})
|
|
654
666
|
unless Cnvrg::CLI.is_response_success(res, false)
|
|
655
667
|
raise Exception.new("Cant download files from the server.")
|
|
@@ -657,6 +669,14 @@ module Cnvrg
|
|
|
657
669
|
self.download_multpile_files_s3(res['result'], @project_home, postfix: postfix, progress: progress)
|
|
658
670
|
end
|
|
659
671
|
|
|
672
|
+
|
|
673
|
+
def download_files_old(files, commit, postfix: '', progress: nil)
|
|
674
|
+
files.each do |file|
|
|
675
|
+
self.download_file_s3(file, commit, postfix: postfix)
|
|
676
|
+
progress.progress += 1 if progress.present?
|
|
677
|
+
end
|
|
678
|
+
end
|
|
679
|
+
|
|
660
680
|
def delete_files_local(deleted, conflicted: [], progress: nil)
|
|
661
681
|
deleted -= conflicted
|
|
662
682
|
deleted.each{|file| self.delete(file); progress.progress += 1 if progress.present?}
|
data/lib/cnvrg/project.rb
CHANGED
|
@@ -11,7 +11,7 @@ module Cnvrg
|
|
|
11
11
|
begin
|
|
12
12
|
@local_path = project_home
|
|
13
13
|
@working_dir = project_home
|
|
14
|
-
config = YAML.load_file(project_home+"/.cnvrg/config.yml")
|
|
14
|
+
config = YAML.load_file(project_home + "/.cnvrg/config.yml")
|
|
15
15
|
@title = config[:project_name]
|
|
16
16
|
@slug = config[:project_slug]
|
|
17
17
|
@owner = config[:owner]
|
|
@@ -45,7 +45,7 @@ module Cnvrg
|
|
|
45
45
|
end
|
|
46
46
|
list = new_ignore.split(",")
|
|
47
47
|
begin
|
|
48
|
-
File.open(self.local_path+"/.cnvrgignore", "a+") do |f|
|
|
48
|
+
File.open(self.local_path + "/.cnvrgignore", "a+") do |f|
|
|
49
49
|
f.puts("\n")
|
|
50
50
|
list.each do |i|
|
|
51
51
|
f.puts("#{i}\n")
|
|
@@ -59,16 +59,16 @@ module Cnvrg
|
|
|
59
59
|
|
|
60
60
|
def get_ignore_list
|
|
61
61
|
ignore_list = []
|
|
62
|
-
if !File.exist? self.local_path+"/.cnvrgignore"
|
|
62
|
+
if !File.exist? self.local_path + "/.cnvrgignore"
|
|
63
63
|
return ignore_list
|
|
64
64
|
end
|
|
65
|
-
File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
|
|
65
|
+
File.open(self.local_path + "/.cnvrgignore", "r").each_line do |line|
|
|
66
66
|
line = line.strip
|
|
67
67
|
if line.start_with? "#" or ignore_list.include? line or line.empty?
|
|
68
68
|
next
|
|
69
69
|
end
|
|
70
70
|
if line.ends_with? "*"
|
|
71
|
-
list_regex = Dir.glob("**/#{line}",File::FNM_DOTMATCH).flatten
|
|
71
|
+
list_regex = Dir.glob("**/#{line}", File::FNM_DOTMATCH).flatten
|
|
72
72
|
list_regex.each do |l|
|
|
73
73
|
ignore_list << l
|
|
74
74
|
if File.directory?(l)
|
|
@@ -79,15 +79,15 @@ module Cnvrg
|
|
|
79
79
|
|
|
80
80
|
end
|
|
81
81
|
elsif line.ends_with? "/*"
|
|
82
|
-
line = line.gsub("/*","")
|
|
82
|
+
line = line.gsub("/*", "")
|
|
83
83
|
regex_list = Dir.glob("**/#{line}/**/*", File::FNM_DOTMATCH).flatten
|
|
84
84
|
ignore_list << regex_list
|
|
85
85
|
elsif line.include? "*"
|
|
86
|
-
|
|
87
|
-
|
|
86
|
+
regex_list = Dir.glob("**/#{line}").flatten
|
|
87
|
+
ignore_list << regex_list
|
|
88
88
|
elsif line.end_with? "/" or File.directory?(line)
|
|
89
|
-
|
|
90
|
-
|
|
89
|
+
ignore_list << line
|
|
90
|
+
all_sub = Dir.glob("#{line}/**/*", File::FNM_DOTMATCH).flatten
|
|
91
91
|
|
|
92
92
|
ignore_list << all_sub.flatten
|
|
93
93
|
|
|
@@ -102,16 +102,16 @@ module Cnvrg
|
|
|
102
102
|
def send_ignore_list()
|
|
103
103
|
begin
|
|
104
104
|
ignore_list = []
|
|
105
|
-
File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
|
|
105
|
+
File.open(self.local_path + "/.cnvrgignore", "r").each_line do |line|
|
|
106
106
|
line = line.strip
|
|
107
107
|
if line.start_with? "#" or ignore_list.include? line or line.empty?
|
|
108
108
|
next
|
|
109
109
|
end
|
|
110
110
|
if line.end_with? "/"
|
|
111
|
-
ignore_list << line.gsub("/","")
|
|
112
|
-
ignore_list << line+"."
|
|
111
|
+
ignore_list << line.gsub("/", "")
|
|
112
|
+
ignore_list << line + "."
|
|
113
113
|
elsif line.include? "*"
|
|
114
|
-
line = line.gsub("*",".*")
|
|
114
|
+
line = line.gsub("*", ".*")
|
|
115
115
|
ignore_list << line
|
|
116
116
|
else
|
|
117
117
|
ignore_list << line
|
|
@@ -126,7 +126,7 @@ module Cnvrg
|
|
|
126
126
|
|
|
127
127
|
# Create project
|
|
128
128
|
|
|
129
|
-
def self.create(project_name, clean, with_docker=false)
|
|
129
|
+
def self.create(project_name, clean, with_docker = false)
|
|
130
130
|
if clean
|
|
131
131
|
list_dirs = [project_name, project_name + "/.cnvrg"]
|
|
132
132
|
else
|
|
@@ -149,7 +149,7 @@ module Cnvrg
|
|
|
149
149
|
]
|
|
150
150
|
cnvrgreadme = Helpers.readme_content
|
|
151
151
|
cnvrgignore = Helpers.cnvrgignore_content
|
|
152
|
-
cnvrghyper
|
|
152
|
+
cnvrghyper = Helpers.hyper_content
|
|
153
153
|
|
|
154
154
|
begin
|
|
155
155
|
|
|
@@ -166,10 +166,10 @@ module Cnvrg
|
|
|
166
166
|
FileUtils.mkdir_p list_dirs
|
|
167
167
|
FileUtils.touch list_files
|
|
168
168
|
|
|
169
|
-
File.open(project_name + "/.cnvrg/config.yml", "w+") {
|
|
170
|
-
File.open(project_name + "/.cnvrgignore", "w+") {
|
|
171
|
-
File.open(project_name + "/README.md", "w+") {
|
|
172
|
-
File.open(project_name + "/src/hyper.yaml", "w+") {
|
|
169
|
+
File.open(project_name + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
|
|
170
|
+
File.open(project_name + "/.cnvrgignore", "w+") {|f| f.write cnvrgignore}
|
|
171
|
+
File.open(project_name + "/README.md", "w+") {|f| f.write cnvrgreadme}
|
|
172
|
+
File.open(project_name + "/src/hyper.yaml", "w+") {|f| f.write cnvrghyper}
|
|
173
173
|
|
|
174
174
|
rescue
|
|
175
175
|
return false
|
|
@@ -177,75 +177,76 @@ module Cnvrg
|
|
|
177
177
|
return true
|
|
178
178
|
end
|
|
179
179
|
|
|
180
|
-
def self.link(owner, project_name, docker=false, git = false)
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
180
|
+
def self.link(owner, project_name, docker = false, git = false)
|
|
181
|
+
ignore_exits = File.exist? ".cnvrgignore"
|
|
182
|
+
list_dirs = [".cnvrg"
|
|
183
|
+
]
|
|
184
|
+
list_files = [
|
|
185
|
+
".cnvrg/config.yml"
|
|
186
|
+
]
|
|
187
|
+
if !ignore_exits
|
|
188
|
+
list_files <<
|
|
189
|
+
".cnvrgignore"
|
|
190
|
+
end
|
|
191
191
|
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
config = {project_name: project_name,
|
|
201
|
-
project_slug: project_slug,
|
|
202
|
-
owner: owner,
|
|
203
|
-
git: git}
|
|
204
|
-
FileUtils.mkdir_p list_dirs
|
|
205
|
-
FileUtils.touch list_files
|
|
206
|
-
File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
|
207
|
-
File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless ignore_exits
|
|
208
|
-
if !File.exist? "README" and !File.exist? "README.md"
|
|
209
|
-
FileUtils.touch [ "README.md" ]
|
|
210
|
-
File.open("README.md", "w+") { |f| f.write cnvrgreadme }
|
|
211
|
-
end
|
|
192
|
+
cnvrgreadme = Helpers.readme_content
|
|
193
|
+
cnvrgignore = Helpers.cnvrgignore_content
|
|
194
|
+
begin
|
|
195
|
+
response = Cnvrg::API.request("cli/create_project", 'POST', {title: project_name, owner: owner, is_docker: docker})
|
|
196
|
+
Cnvrg::CLI.is_response_success(response)
|
|
197
|
+
response = JSON.parse response["result"]
|
|
198
|
+
project_slug = response["slug"]
|
|
212
199
|
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
200
|
+
config = {project_name: project_name,
|
|
201
|
+
project_slug: project_slug,
|
|
202
|
+
owner: owner,
|
|
203
|
+
git: git}
|
|
204
|
+
FileUtils.mkdir_p list_dirs
|
|
205
|
+
FileUtils.touch list_files
|
|
206
|
+
File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
|
|
207
|
+
File.open(".cnvrgignore", "w+") {|f| f.write cnvrgignore} unless ignore_exits
|
|
208
|
+
if !File.exist? "README" and !File.exist? "README.md"
|
|
209
|
+
FileUtils.touch ["README.md"]
|
|
210
|
+
File.open("README.md", "w+") {|f| f.write cnvrgreadme}
|
|
216
211
|
end
|
|
217
|
-
|
|
212
|
+
|
|
213
|
+
rescue => e
|
|
214
|
+
puts e
|
|
215
|
+
return false
|
|
218
216
|
end
|
|
217
|
+
return true
|
|
218
|
+
end
|
|
219
219
|
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
220
|
+
def self.clone_dir(project_slug, project_owner, project_name, is_git = false)
|
|
221
|
+
list_dirs = [project_name,
|
|
222
|
+
project_name + "/.cnvrg"
|
|
223
|
+
]
|
|
224
224
|
|
|
225
225
|
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
226
|
+
list_files = [
|
|
227
|
+
project_name + "/.cnvrg/config.yml",
|
|
228
|
+
project_name + "/.cnvrgignore",
|
|
229
|
+
]
|
|
230
|
+
begin
|
|
231
|
+
config = {project_name: project_name,
|
|
232
|
+
project_slug: project_slug,
|
|
233
|
+
owner: project_owner,
|
|
234
|
+
git: is_git}
|
|
235
|
+
FileUtils.mkdir_p list_dirs
|
|
236
|
+
FileUtils.touch list_files
|
|
237
|
+
cnvrgignore = Helpers.cnvrgignore_content
|
|
238
238
|
|
|
239
239
|
|
|
240
|
-
|
|
241
|
-
|
|
240
|
+
File.open(project_name + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
|
|
241
|
+
File.open(project_name + "/.cnvrgignore", "w+") {|f| f.write cnvrgignore}
|
|
242
242
|
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
end
|
|
246
|
-
return true
|
|
243
|
+
rescue
|
|
244
|
+
return false
|
|
247
245
|
end
|
|
248
|
-
|
|
246
|
+
return true
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
def self.verify_cnvrgignore_exist(project_name, remote)
|
|
249
250
|
path = ".cnvrgignore"
|
|
250
251
|
if !File.exist? path
|
|
251
252
|
path = "#{project_name}/.cnvrgignore"
|
|
@@ -255,10 +256,10 @@ module Cnvrg
|
|
|
255
256
|
begin
|
|
256
257
|
list_files = [
|
|
257
258
|
path
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
259
|
+
]
|
|
260
|
+
FileUtils.touch list_files
|
|
261
|
+
cnvrgignore = Helpers.cnvrgignore_content
|
|
262
|
+
File.open(path, "w+") {|f| f.write cnvrgignore}
|
|
262
263
|
rescue => e
|
|
263
264
|
return false
|
|
264
265
|
end
|
|
@@ -266,64 +267,66 @@ module Cnvrg
|
|
|
266
267
|
end
|
|
267
268
|
end
|
|
268
269
|
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
270
|
+
def self.clone_dir_remote(project_slug, project_owner, project_name, is_git = false)
|
|
271
|
+
list_dirs = [
|
|
272
|
+
".cnvrg"
|
|
273
|
+
]
|
|
275
274
|
|
|
276
|
-
list_files = [
|
|
277
|
-
".cnvrg/config.yml",
|
|
278
275
|
|
|
279
|
-
|
|
276
|
+
list_files = [
|
|
277
|
+
".cnvrg/config.yml",
|
|
280
278
|
|
|
279
|
+
]
|
|
281
280
|
|
|
282
|
-
begin
|
|
283
|
-
config = {project_name: project_name,
|
|
284
|
-
project_slug: project_slug,
|
|
285
|
-
owner: project_owner,
|
|
286
|
-
git: is_git
|
|
287
|
-
}
|
|
288
|
-
FileUtils.mkdir_p list_dirs
|
|
289
|
-
FileUtils.touch list_files
|
|
290
281
|
|
|
282
|
+
begin
|
|
283
|
+
config = {project_name: project_name,
|
|
284
|
+
project_slug: project_slug,
|
|
285
|
+
owner: project_owner,
|
|
286
|
+
git: is_git
|
|
287
|
+
}
|
|
288
|
+
FileUtils.mkdir_p list_dirs
|
|
289
|
+
FileUtils.touch list_files
|
|
291
290
|
|
|
292
|
-
File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
|
293
|
-
if !File.exist? ".cnvrgignore"
|
|
294
|
-
FileUtils.touch ".cnvrgignore"
|
|
295
|
-
list_files << ".cnvrgignore"
|
|
296
|
-
cnvrgignore = Helpers.cnvrgignore_content
|
|
297
|
-
File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore }
|
|
298
291
|
|
|
292
|
+
File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
|
|
293
|
+
if !File.exist? ".cnvrgignore"
|
|
294
|
+
FileUtils.touch ".cnvrgignore"
|
|
295
|
+
list_files << ".cnvrgignore"
|
|
296
|
+
cnvrgignore = Helpers.cnvrgignore_content
|
|
297
|
+
File.open(".cnvrgignore", "w+") {|f| f.write cnvrgignore}
|
|
299
298
|
|
|
300
|
-
end
|
|
301
299
|
|
|
302
|
-
rescue
|
|
303
|
-
return false
|
|
304
300
|
end
|
|
305
|
-
|
|
301
|
+
|
|
302
|
+
rescue
|
|
303
|
+
return false
|
|
306
304
|
end
|
|
305
|
+
return true
|
|
306
|
+
end
|
|
307
|
+
|
|
307
308
|
def update_is_new_branch(new_branch)
|
|
308
|
-
config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
|
|
309
|
+
config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
|
|
309
310
|
config[:new_branch] = new_branch
|
|
310
|
-
File.open(@working_dir+"/.cnvrg/config.yml", "w+") {
|
|
311
|
+
File.open(@working_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
|
|
311
312
|
end
|
|
313
|
+
|
|
312
314
|
def get_new_branch
|
|
313
315
|
begin
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
rescue =>e
|
|
316
|
+
config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
|
|
317
|
+
return config[:new_branch]
|
|
318
|
+
rescue => e
|
|
317
319
|
return false
|
|
318
320
|
|
|
319
321
|
end
|
|
320
322
|
end
|
|
321
323
|
|
|
322
324
|
def remove_new_branch
|
|
323
|
-
config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
|
|
325
|
+
config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
|
|
324
326
|
new_config = config.except(:new_branch)
|
|
325
|
-
File.open(@working_dir+"/.cnvrg/config.yml", "w+") {
|
|
327
|
+
File.open(@working_dir + "/.cnvrg/config.yml", "w+") {|f| f.write new_config.to_yaml}
|
|
326
328
|
end
|
|
329
|
+
|
|
327
330
|
def generate_output_dir(output_dir)
|
|
328
331
|
Cnvrg::Logger.log_info("Generating output dir for #{output_dir}")
|
|
329
332
|
upload_list = []
|
|
@@ -332,10 +335,10 @@ module Cnvrg
|
|
|
332
335
|
next if e.end_with? "/."
|
|
333
336
|
if File.directory? e
|
|
334
337
|
|
|
335
|
-
upload_list << e+"/"
|
|
338
|
+
upload_list << e + "/"
|
|
336
339
|
else
|
|
337
340
|
upload_list << e
|
|
338
|
-
|
|
341
|
+
end
|
|
339
342
|
end
|
|
340
343
|
if Dir.exists? output_dir
|
|
341
344
|
upload_list << output_dir + "/"
|
|
@@ -364,130 +367,132 @@ module Cnvrg
|
|
|
364
367
|
|
|
365
368
|
end
|
|
366
369
|
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
370
|
+
def generate_idx(deploy: false)
|
|
371
|
+
if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
|
|
372
|
+
old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
|
373
|
+
else
|
|
374
|
+
old_idx = {:tree => {}, :commit => nil}
|
|
375
|
+
end
|
|
373
376
|
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
377
|
+
tree_idx = Hash.new(0)
|
|
378
|
+
list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject {|x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~ /^#{self.local_path}\/\.git\/*/) or (x =~ /^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~ /^#{self.local_path}\/\.cnvrgignore/)}
|
|
379
|
+
list_ignore = self.get_ignore_list()
|
|
380
|
+
if deploy
|
|
381
|
+
list_ignore << ["main.py", "main.pyc", "__init__.py", "uwsgi.ini"]
|
|
382
|
+
list_ignore.flatten!
|
|
383
|
+
end
|
|
381
384
|
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
385
|
+
Parallel.map(list, in_threads: IDXParallelThreads) do |e|
|
|
386
|
+
label = e.gsub(self.local_path + "/", "")
|
|
387
|
+
ignore_label = label.gsub("/", "//")
|
|
388
|
+
if list_ignore.include? ignore_label
|
|
389
|
+
next
|
|
390
|
+
end
|
|
391
|
+
if File.directory? e
|
|
389
392
|
|
|
390
|
-
|
|
393
|
+
tree_idx[label + "/"] = nil
|
|
394
|
+
else
|
|
395
|
+
sha1 = OpenSSL::Digest::SHA1.file(e).hexdigest
|
|
396
|
+
if old_idx.nil? or old_idx.to_h[:tree].nil?
|
|
397
|
+
tree_idx[label] = {sha1: sha1, commit_time: nil}
|
|
398
|
+
elsif old_idx[:tree][label].nil? or old_idx[:tree][label][:sha1] != sha1
|
|
399
|
+
tree_idx[label] = {sha1: sha1, commit_time: nil}
|
|
391
400
|
else
|
|
392
|
-
|
|
393
|
-
if old_idx.nil? or old_idx.to_h[:tree].nil?
|
|
394
|
-
tree_idx[label] = {sha1: sha1, commit_time: nil}
|
|
395
|
-
elsif old_idx[:tree][label].nil? or old_idx[:tree][label][:sha1] != sha1
|
|
396
|
-
tree_idx[label] = {sha1: sha1, commit_time: nil}
|
|
397
|
-
else
|
|
398
|
-
tree_idx[label] = old_idx[:tree][label]
|
|
399
|
-
end
|
|
401
|
+
tree_idx[label] = old_idx[:tree][label]
|
|
400
402
|
end
|
|
401
403
|
end
|
|
404
|
+
end
|
|
402
405
|
|
|
403
|
-
|
|
406
|
+
old_idx[:tree] = tree_idx
|
|
404
407
|
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
|
408
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write old_idx.to_yaml}
|
|
409
|
+
return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
def get_idx
|
|
413
|
+
unless File.exists? "#{self.local_path}/.cnvrg/idx.yml"
|
|
414
|
+
empty_idx = {:commit => nil, :tree => {}}
|
|
415
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write empty_idx.to_yaml}
|
|
416
|
+
return empty_idx
|
|
415
417
|
end
|
|
418
|
+
YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
|
419
|
+
end
|
|
416
420
|
|
|
417
421
|
def set_idx(idx)
|
|
418
422
|
FileUtils.mkdir_p("#{self.local_path}/.cnvrg")
|
|
419
|
-
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {
|
|
423
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx.to_yaml}
|
|
424
|
+
end
|
|
425
|
+
|
|
426
|
+
def clone(remote = 0, commit)
|
|
427
|
+
response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/clone", 'POST', {project_slug: self.slug, remote: remote, commit: commit})
|
|
428
|
+
return response
|
|
420
429
|
end
|
|
421
430
|
|
|
422
|
-
def clone(remote=0, commit)
|
|
423
|
-
response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/clone", 'POST', {project_slug: self.slug, remote: remote, commit: commit})
|
|
424
|
-
return response
|
|
425
|
-
end
|
|
426
431
|
def git_download_commit(commit)
|
|
427
432
|
response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/git_download_commit", 'POST', {commit_sha1: commit})
|
|
428
|
-
CLI.is_response_success(response,true)
|
|
433
|
+
CLI.is_response_success(response, true)
|
|
429
434
|
return response
|
|
430
435
|
end
|
|
431
436
|
|
|
432
437
|
def compare_idx(new_branch, force:false, deploy: false, in_exp:false, specific_files: [], download: false)
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
end
|
|
451
|
-
response ={"result"=> {"commit"=>nil,"tree"=> {"added"=> added,
|
|
452
|
-
"updated_on_server"=> [],
|
|
453
|
-
"updated_on_local"=> [],
|
|
454
|
-
"update_local" => [],
|
|
455
|
-
"deleted"=> [],
|
|
456
|
-
"conflicts"=> []} } }
|
|
457
|
-
return response
|
|
438
|
+
is_download = download
|
|
439
|
+
if is_download
|
|
440
|
+
local_idx = self.get_idx
|
|
441
|
+
else
|
|
442
|
+
#upload
|
|
443
|
+
local_idx = self.generate_idx(deploy: deploy)
|
|
444
|
+
end
|
|
445
|
+
commit = local_idx[:commit]
|
|
446
|
+
tree = local_idx[:tree]
|
|
447
|
+
ignore_list = self.send_ignore_list()
|
|
448
|
+
if force or specific_files.present?
|
|
449
|
+
added = []
|
|
450
|
+
if specific_files.present?
|
|
451
|
+
added = specific_files
|
|
452
|
+
elsif tree.present?
|
|
453
|
+
added << local_idx[:tree].keys
|
|
454
|
+
added.flatten!
|
|
458
455
|
end
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
456
|
+
response = {"result" => {"commit" => nil, "tree" => {"added" => added,
|
|
457
|
+
"updated_on_server" => [],
|
|
458
|
+
"updated_on_local" => [],
|
|
459
|
+
"update_local" => [],
|
|
460
|
+
"deleted" => [],
|
|
461
|
+
"conflicts" => []}}}
|
|
462
|
+
return response
|
|
463
|
+
end
|
|
464
|
+
#we dont want to send it on download - we only compare between commits sha1 in download.
|
|
465
|
+
if is_download
|
|
466
|
+
#the new server doesnt need the tree, but the old probably needs :X
|
|
467
|
+
local_idx[:tree] = {} if Cnvrg::Helpers.server_version > 0
|
|
468
|
+
end
|
|
469
|
+
response = Cnvrg::API.request(@base_resource + "status", 'POST', {idx: local_idx, new_branch: new_branch,
|
|
470
|
+
current_commit: commit, ignore: ignore_list, force: force, in_exp: in_exp, download: download})
|
|
471
|
+
|
|
472
|
+
CLI.is_response_success(response, true)
|
|
473
|
+
if is_download
|
|
474
|
+
if Cnvrg::Helpers.server_version > 0
|
|
475
|
+
#trying to optimize the download using resolver
|
|
476
|
+
resolve = response['result']['tree']['resolver'] || tree #tree of file -> sha1 from current commit to check conflicts
|
|
477
|
+
destination_files = response['result']['tree']['destination'] || {} #tree of file -> sha1 from last commit to check files that already downloaded
|
|
478
|
+
else
|
|
479
|
+
resolve = tree
|
|
480
|
+
destination_files = {}
|
|
463
481
|
end
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
CLI.is_response_success(response,true)
|
|
468
|
-
if is_download
|
|
469
|
-
if Cnvrg::Helpers.server_version > 0
|
|
470
|
-
#trying to optimize the download using resolver
|
|
471
|
-
resolve = response['result']['tree']['resolver'] || tree #tree of file -> sha1 from current commit to check conflicts
|
|
472
|
-
destination_files = response['result']['tree']['destination'] || {} #tree of file -> sha1 from last commit to check files that already downloaded
|
|
473
|
-
else
|
|
474
|
-
resolve = tree
|
|
475
|
-
destination_files = {}
|
|
476
|
-
end
|
|
477
|
-
@files = self.get_files
|
|
478
|
-
local_tree = @files.calculate_sha1(resolve.keys)
|
|
479
|
-
changed_files = resolve.keys.select{|file| local_tree[file] != resolve[file]}
|
|
482
|
+
@files = self.get_files
|
|
483
|
+
local_tree = @files.calculate_sha1(resolve.keys)
|
|
484
|
+
changed_files = resolve.keys.select {|file| local_tree[file] != resolve[file]}
|
|
480
485
|
|
|
481
|
-
|
|
482
|
-
|
|
486
|
+
# means that the user changed the file locally
|
|
487
|
+
response['result']['tree']['update_local'] = changed_files
|
|
483
488
|
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
end
|
|
489
|
-
return response
|
|
489
|
+
# means that we already downloaded this file and we dont need it anymore
|
|
490
|
+
downloaded_files = destination_files.keys.select {|file| local_tree[file] == destination_files[file]}
|
|
491
|
+
response['result']['tree']['added'] -= downloaded_files
|
|
492
|
+
response['result']['tree']['updated_on_server'] -= downloaded_files
|
|
490
493
|
end
|
|
494
|
+
return response
|
|
495
|
+
end
|
|
491
496
|
|
|
492
497
|
def get_files
|
|
493
498
|
Cnvrg::Files.new(self.owner, self.slug, project_home: @local_path)
|
|
@@ -502,52 +507,52 @@ module Cnvrg
|
|
|
502
507
|
'POST',
|
|
503
508
|
{tree: tree, ignore: ignore_list,
|
|
504
509
|
dest_commit: destination, current_commit: current_commit})
|
|
505
|
-
CLI.is_response_success(response,false)
|
|
510
|
+
CLI.is_response_success(response, false)
|
|
506
511
|
response
|
|
507
512
|
end
|
|
508
513
|
|
|
509
514
|
def set_on_branch(is_latest)
|
|
510
|
-
config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
|
|
515
|
+
config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
|
|
511
516
|
@is_branch = !is_latest
|
|
512
517
|
config[:is_branch] = @is_branch
|
|
513
518
|
|
|
514
|
-
File.open(@working_dir+"/.cnvrg/config.yml", 'w') {
|
|
519
|
+
File.open(@working_dir + "/.cnvrg/config.yml", 'w') {|f| f.write config.to_yaml}
|
|
515
520
|
|
|
516
521
|
end
|
|
517
522
|
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
end
|
|
522
|
-
response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
|
|
523
|
-
CLI.is_response_success(response,false)
|
|
524
|
-
update_is_new_branch(response["result"]["new_branch"])
|
|
525
|
-
return response["result"]["new_branch"]
|
|
523
|
+
def compare_commit(commit)
|
|
524
|
+
if commit.nil? or commit.empty?
|
|
525
|
+
commit = last_local_commit
|
|
526
526
|
end
|
|
527
|
+
response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
|
|
528
|
+
CLI.is_response_success(response, false)
|
|
529
|
+
update_is_new_branch(response["result"]["new_branch"])
|
|
530
|
+
return response["result"]["new_branch"]
|
|
531
|
+
end
|
|
527
532
|
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
|
531
|
-
files.each do |path|
|
|
532
|
-
idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
|
|
533
|
-
end
|
|
534
|
-
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
|
|
533
|
+
def update_idx_with_files_commits!(files, commit_time)
|
|
535
534
|
|
|
536
|
-
|
|
535
|
+
idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
|
536
|
+
files.each do |path|
|
|
537
|
+
idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
|
|
537
538
|
end
|
|
539
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
|
|
540
|
+
|
|
541
|
+
return true
|
|
542
|
+
end
|
|
538
543
|
|
|
539
|
-
|
|
544
|
+
def deploy(file_to_run, function, input_params, commit_to_run, instance_type, image_slug, scheduling_query, local_timestamp, workers, file_input, title)
|
|
545
|
+
response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/deploy", 'POST', {file_to_run: file_to_run, function: function,
|
|
546
|
+
image_slug: image_slug, input_params: input_params,
|
|
547
|
+
commit_sha1: commit_to_run,
|
|
548
|
+
instance_type: instance_type,
|
|
549
|
+
scheduling_query: scheduling_query,
|
|
550
|
+
local_timestamp: local_timestamp,
|
|
551
|
+
workers: workers, file_input: file_input,
|
|
552
|
+
title: title})
|
|
553
|
+
return response
|
|
554
|
+
end
|
|
540
555
|
|
|
541
|
-
response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/deploy", 'POST', {file_to_run: file_to_run, function: function,
|
|
542
|
-
image_slug: image_slug, input_params: input_params,
|
|
543
|
-
commit_sha1: commit_to_run,
|
|
544
|
-
instance_type: instance_type,
|
|
545
|
-
scheduling_query: scheduling_query,
|
|
546
|
-
local_timestamp: local_timestamp,
|
|
547
|
-
workers:workers,file_input:file_input,
|
|
548
|
-
title: title})
|
|
549
|
-
return response
|
|
550
|
-
end
|
|
551
556
|
def list_commits
|
|
552
557
|
response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commits/list", 'GET')
|
|
553
558
|
CLI.is_response_success(response)
|
|
@@ -567,17 +572,19 @@ module Cnvrg
|
|
|
567
572
|
CLI.is_response_success(response)
|
|
568
573
|
return response
|
|
569
574
|
end
|
|
570
|
-
def update_idx_with_commit!(commit)
|
|
571
|
-
idx_hash = self.get_idx
|
|
572
|
-
idx_hash[:commit] = commit
|
|
573
575
|
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
576
|
+
def update_idx_with_commit!(commit, latest: nil)
|
|
577
|
+
idx_hash = self.get_idx
|
|
578
|
+
idx_hash[:commit] = commit
|
|
579
|
+
self.set_on_branch(latest) unless latest.nil?
|
|
577
580
|
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
|
|
582
|
+
return true
|
|
583
|
+
end
|
|
584
|
+
|
|
585
|
+
def revert(working_dir)
|
|
586
|
+
FileUtils.rm_rf working_dir
|
|
587
|
+
end
|
|
581
588
|
|
|
582
589
|
def init_machines
|
|
583
590
|
Cnvrg::Logger.log_info("Init machines.")
|
|
@@ -599,5 +606,6 @@ module Cnvrg
|
|
|
599
606
|
machines.include? machine
|
|
600
607
|
end
|
|
601
608
|
|
|
602
|
-
|
|
609
|
+
|
|
603
610
|
end
|
|
611
|
+
end
|
data/lib/cnvrg/version.rb
CHANGED