cnvrg 0.8.5 → 0.8.6

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: cb82613324f1a1f409c0819b795be4a580bf6f31
4
- data.tar.gz: fb950e5920d8cae1cd10609ec4ba0eba1707bfbf
3
+ metadata.gz: 819730717280428ac1ba231d43462cb827253cc1
4
+ data.tar.gz: 95c1963b178296af0655d54b691b02895ced41ef
5
5
  SHA512:
6
- metadata.gz: 3d51178739e2851c1871de672d85ca1c69d099cdb9f94f3b80a222ae1cc30cbfae60a9d1892995be0657c9cf5f8eb406cbd9ae30cda3a4183891a87370f226d0
7
- data.tar.gz: 3cb42c96a9e8a04e7a0ee672e5ac404ae46b4668b96e6edc107ee89d46a48e64764f5b46cabfb163e060b00b3982fb5a7a7ac7755d56d9aaf72bb0b9e8a9e01f
6
+ metadata.gz: 61563035b65f94b7d87354a6dcb783119179edf655cd6881595035417d15c6441b0337f48ae9de9b410136bd3133eb3254fb995af46ffbd67d785e09312aa3e2
7
+ data.tar.gz: fda3bfcd38199e6c04c46a2c2ca7812182e58352c0a0c9d06898acca3f5de30178160d35ab6b6d86d640d7c89ecb1a773a562d44af369e652b5e476943aa8c21
@@ -305,7 +305,7 @@ module Cnvrg
305
305
  compression_path = "#{home_dir}/.cnvrg/tmp"
306
306
  config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url, compression_path: compression_path}
307
307
  File.open(home_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
308
- say "Done"
308
+ say "Done", Thor::Shell::Color::GREEN
309
309
  rescue
310
310
  say "ERROR", Thor::Shell::Color::RED
311
311
  File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
@@ -340,7 +340,7 @@ module Cnvrg
340
340
  compression_path = "#{home_dir}/.cnvrg/tmp"
341
341
  config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url, compression_path: compression_path}
342
342
  File.open(home_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
343
- say "Done"
343
+ say "Done", Thor::Shell::Color::GREEN
344
344
  rescue
345
345
  say "ERROR", Thor::Shell::Color::RED
346
346
  end
@@ -408,7 +408,7 @@ module Cnvrg
408
408
  end
409
409
  end
410
410
  rescue SignalException
411
- say "\nAborting"
411
+ say "\nAborting", Thor::Shell::Color::RED
412
412
  exit(1)
413
413
  end
414
414
  end
@@ -442,7 +442,7 @@ module Cnvrg
442
442
  say "#{checks} Done", Thor::Shell::Color::GREEN
443
443
 
444
444
  rescue SignalException
445
- say "\nAborting"
445
+ say "\nAborting", Thor::Shell::Color::RED
446
446
  exit(1)
447
447
  end
448
448
  end
@@ -544,7 +544,7 @@ module Cnvrg
544
544
  exit(1)
545
545
  rescue SignalException
546
546
 
547
- say "\nAborting"
547
+ say "\nAborting",Thor::Shell::Color::RED
548
548
  logout()
549
549
  exit(1)
550
550
  end
@@ -565,7 +565,7 @@ module Cnvrg
565
565
  puts e.message
566
566
  puts e.backtrace
567
567
  rescue SignalException
568
- say "\nAborting"
568
+ say "\nAborting",Thor::Shell::Color::RED
569
569
  exit(1)
570
570
  end
571
571
 
@@ -587,7 +587,7 @@ module Cnvrg
587
587
 
588
588
 
589
589
  rescue SignalException
590
- say "\nAborting"
590
+ say "\nAborting", Thor::Shell::Color::RED
591
591
  exit(1)
592
592
  end
593
593
  end
@@ -676,7 +676,7 @@ module Cnvrg
676
676
 
677
677
  @project.revert(working_dir)
678
678
  end
679
- say "\nAborting"
679
+ say "\nAborting", Thor::Shell::Color::RED
680
680
  exit(1)
681
681
  end
682
682
  end
@@ -762,7 +762,7 @@ module Cnvrg
762
762
  log_error(e)
763
763
  rescue SignalException
764
764
 
765
- say "\nAborting"
765
+ say "\nAborting", Thor::Shell::Color::RED
766
766
  exit(1)
767
767
  end
768
768
  end
@@ -802,7 +802,7 @@ module Cnvrg
802
802
  rescue => e
803
803
  log_error(e)
804
804
  rescue SignalException
805
- say "\nAborting"
805
+ say "\nAborting", Thor::Shell::Color::RED
806
806
  exit(1)
807
807
  end
808
808
  end
@@ -826,7 +826,7 @@ module Cnvrg
826
826
  log_error(e)
827
827
  rescue SignalException
828
828
 
829
- say "\nAborting"
829
+ say "\nAborting", Thor::Shell::Color::RED
830
830
  exit(1)
831
831
  end
832
832
  end
@@ -891,6 +891,7 @@ module Cnvrg
891
891
  downloaded_files += files['keys'].length
892
892
  files = @files.get_clone_chunk(commit: commit, latest_id: files['latest'])
893
893
  end
894
+ progressbar.finish
894
895
  if downloaded_files == files_count
895
896
  Dataset.verify_cnvrgignore_exist(dataset_name, false)
896
897
  @dataset = Dataset.new(dataset_home)
@@ -905,7 +906,7 @@ module Cnvrg
905
906
  exit(1)
906
907
  end
907
908
  rescue SignalException
908
- say "\nAborting"
909
+ say "\nAborting", Thor::Shell::Color::RED
909
910
  exit(1)
910
911
  end
911
912
  end
@@ -983,7 +984,7 @@ module Cnvrg
983
984
  end
984
985
  end
985
986
  rescue SignalException
986
- say "\nAborting"
987
+ say "\nAborting", Thor::Shell::Color::RED
987
988
  exit(1)
988
989
  end
989
990
  end
@@ -1009,7 +1010,7 @@ module Cnvrg
1009
1010
 
1010
1011
  rescue SignalException
1011
1012
 
1012
- say "\nAborting"
1013
+ say "\nAborting", Thor::Shell::Color::RED
1013
1014
  exit(1)
1014
1015
  end
1015
1016
  end
@@ -1031,7 +1032,7 @@ module Cnvrg
1031
1032
  rescue SignalException
1032
1033
  log_end(-1)
1033
1034
 
1034
- say "\nAborting"
1035
+ say "\nAborting", Thor::Shell::Color::RED
1035
1036
  exit(1)
1036
1037
  end
1037
1038
  end
@@ -1813,112 +1814,22 @@ module Cnvrg
1813
1814
 
1814
1815
  if clone_resp
1815
1816
  @project = Project.new(project_home)
1816
- @files = Cnvrg::Files.new(@project.owner, slug)
1817
+ @files = Cnvrg::Files.new(@project.owner, slug, project_home: project_home)
1817
1818
  response = @project.clone(remote, commit_to_clone)
1818
1819
  Cnvrg::CLI.is_response_success response
1820
+ commit_sha1 = response["result"]["commit"]
1821
+ files = response["result"]["tree"].keys
1819
1822
  idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
1820
-
1821
1823
  log_message("Downloading files", Thor::Shell::Color::BLUE)
1822
- files = response["result"]["files"]
1823
-
1824
- if !files.nil?
1825
- begin
1826
- download_result = @files.download_multpile_files_s3(files, project_home)
1827
- @project.set_idx(idx)
1828
-
1829
- rescue Interrupt
1830
- log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
1831
-
1832
- @files.revoke_clone(project_home)
1833
- return
1834
- end
1835
- Project.verify_cnvrgignore_exist(project_name, remote)
1836
- if !download_result.is_success?
1837
- log_message(download_result.e_msg, download_result.msg_color, false)
1838
- log_message(download_result.e_msg_backtrace, download_result.msg_color, false)
1839
-
1840
- @files.revoke_clone(project_home)
1841
- end
1842
- log_message(download_result.msg, download_result.msg_color)
1843
- else
1844
- current_commit = response["result"]["commit"]
1845
- if !response["result"]["tree"].nil?
1846
- parallel_options = {
1847
- :progress => {
1848
- :title => "Download Progress",
1849
- :progress_mark => '=',
1850
- :format => "%b>>%i| %p%% %t",
1851
- :starting_at => 0,
1852
- :total => response["result"]["tree"].size,
1853
- :autofinish => true
1854
- },
1855
- in_processes: ParallelProcesses,
1856
- in_thread: ParallelThreads
1857
- }
1858
- begin
1859
- successful_changes =[]
1860
- is_success = true
1861
- clone_result = Parallel.map((response["result"]["tree"]), parallel_options) do |f|
1862
-
1863
- relative_path = f[0].gsub(/^#{@project.local_path}/, "")
1864
- if f[0].end_with? "/"
1865
- # dir
1866
- if @files.download_dir(f[0], relative_path, project_home)
1867
- f
1868
- successful_changes << relative_path
1869
- else
1870
- is_success =false
1871
- log_message("Could not create directory: #{f[0]}", Thor::Shell::Color::RED)
1872
- raise Parallel::Kill
1873
- end
1874
- else
1875
- # blob
1876
-
1877
- if @files.download_file_s3(f[0], relative_path, project_home, commit_sha1=current_commit)
1878
- f
1879
- successful_changes << relative_path
1880
- else
1881
- is_success =false
1882
- log_message("Could not download file: #{f[0]}", Thor::Shell::Color::RED)
1883
- raise Parallel::Kill
1884
-
1885
- end
1886
- end
1887
- end
1888
- successful_changes = response["result"]["tree"]
1889
- if !successful_changes.nil? and is_success
1890
- @project.set_idx(idx)
1891
- Project.verify_cnvrgignore_exist(project_name,remote)
1892
- log_message("Done.\nDownloaded #{successful_changes.size}/#{response["result"]["tree"].size} files", Thor::Shell::Color::GREEN)
1893
- else
1894
- log_message("Couldn't download some files", Thor::Shell::Color::RED)
1895
-
1896
- end
1897
- rescue Interrupt
1898
- log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
1899
-
1900
- @files.revoke_clone(project_home)
1901
- return
1902
- end
1903
- end
1904
-
1905
-
1906
- end
1907
-
1908
- else
1909
-
1910
- log_message("Error: Couldn't clone: #{project_name}", Thor::Shell::Color::RED)
1911
- @files.revoke_clone(project_home)
1912
-
1913
- return
1824
+ progressbar = @files.create_progressbar(files.size, "Clone Progress")
1825
+ @files.download_files(files, commit_sha1, progress: progressbar)
1826
+ progressbar.finish
1827
+ Project.verify_cnvrgignore_exist(project_name, remote)
1828
+ @project.set_idx(idx)
1829
+ log_message("Done")
1830
+ log_message("Downloaded #{files.size} files")
1914
1831
  end
1915
- rescue SignalException
1916
- say "\nAborting"
1917
- @files.revoke_clone(project_home)
1918
-
1919
- return
1920
1832
  end
1921
-
1922
1833
  end
1923
1834
 
1924
1835
 
@@ -2360,7 +2271,7 @@ module Cnvrg
2360
2271
  :autofinish => true)
2361
2272
  @files.upload_multiple_files(to_upload, commit_sha1, progress: progressbar)
2362
2273
  @files.delete_files_from_server(deleted, commit_sha1)
2363
- progressbar.progress += deleted.size
2274
+ progressbar.finish
2364
2275
  res = @files.end_commit(commit_sha1, force: force, message: commit_msg)
2365
2276
  unless Cnvrg::CLI.is_response_success(res, false)
2366
2277
  raise StandardError.new("Cant end commit")
@@ -2614,6 +2525,9 @@ module Cnvrg
2614
2525
  if git or @project.is_git
2615
2526
  return download_in_git(commit)
2616
2527
  end
2528
+ if commit.present?
2529
+ return jump(commit)
2530
+ end
2617
2531
  ignore = options[:ignore] || ""
2618
2532
  if ignore.nil? or ignore.empty?
2619
2533
  ignore = ignore_list
@@ -2683,17 +2597,20 @@ module Cnvrg
2683
2597
 
2684
2598
 
2685
2599
  Cnvrg::Logger.log_info("Downloading updated files:#{updated_files.join(",")}")
2686
- @files.download_files(updated_files, commit, progress: progressbar) if updated_files.present?
2600
+ @files.download_files(updated_files, commit, progress: progressbar)
2687
2601
 
2688
2602
  Cnvrg::Logger.log_info("Downloading conflicted files:#{conflicted_files.join(",")}")
2689
- @files.download_files(conflicted_files, commit, postfix: ".conflict", progress: progressbar) if conflicted_files.present?
2603
+ @files.download_files(conflicted_files, commit, postfix: ".conflict", progress: progressbar)
2690
2604
 
2691
2605
  Cnvrg::Logger.log_info("Delete files: #{deleted_files.join(",")}")
2692
2606
  @files.delete_files_local(deleted_files, conflicted: conflicted_deleted, progress: progressbar)
2693
2607
 
2694
2608
  # update idx with latest commit
2695
- @project.update_idx_with_commit!(commit)
2609
+ # the latest true its because if we define --commit in the cmd it will go to "def jump(options['commit'])"
2610
+ # so if we are downloads something, we have to stay here.
2611
+ @project.update_idx_with_commit!(commit, latest: true)
2696
2612
  #TODO Sync, remove idx, sync again and pray
2613
+ progressbar.finish
2697
2614
  check = Helpers.checkmark()
2698
2615
  Cnvrg::Logger.log_info("Finished downloading successfuly")
2699
2616
  if options["verbose"]
@@ -2728,10 +2645,11 @@ module Cnvrg
2728
2645
  project_home = get_project_home
2729
2646
  @project = Project.new(project_home)
2730
2647
  current_commit = @project.last_local_commit
2731
- if current_commit.eql? commit_sha1
2648
+ if current_commit.start_with? commit_sha1 #commit_sha1 can be partial.
2732
2649
  log_message("Project is already updated", Thor::Shell::Color::GREEN)
2733
2650
  exit(0)
2734
2651
  end
2652
+ log_message("Jumping to commit #{commit_sha1}")
2735
2653
  @files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: project_home)
2736
2654
  resp = @project.jump_idx(destination: commit_sha1)
2737
2655
  if resp.blank?
@@ -2739,81 +2657,27 @@ module Cnvrg
2739
2657
  exit(0)
2740
2658
  end
2741
2659
  compare = resp['result']['compare']
2742
- resolver = resp['result']['resolver']
2743
2660
  latest = resp['result']['latest']
2744
2661
  commit = resp['result']['commit']
2745
- updated_files = compare['updated_on_server']
2662
+ updated_files = compare['updated_on_server'] + compare['added']
2746
2663
  conflicted_files = compare['conflicts']
2664
+ conflicted_deleted = compare['delete_conflicts'] || []
2747
2665
  deleted_files = compare['deleted']
2748
2666
  overall_changes = [updated_files, conflicted_files, deleted_files].flatten.size
2749
2667
 
2750
-
2751
- progressbar = ProgressBar.create(:title => "Download Progress",
2752
- :progress_mark => '=',
2753
- :format => "%b>>%i| %p%% %t",
2754
- :starting_at => 0,
2755
- :total => overall_changes,
2756
- :autofinish => true)
2757
-
2668
+ progressbar = @files.create_progressbar(overall_changes, "Download Progress")
2758
2669
  @files.download_files(updated_files, commit_sha1, progress: progressbar)
2759
2670
  @files.download_files(conflicted_files, commit_sha1, progress: progressbar, postfix: '.conflicted')
2760
- @files.delete_files_local(deleted_files, progress: progressbar)
2761
-
2762
-
2763
- @project.set_on_branch(latest)
2764
- @project.update_idx_with_commit!(commit)
2671
+ @files.delete_files_local(deleted_files, progress: progressbar, conflicted: conflicted_deleted)
2672
+ progressbar.finish
2673
+ @project.update_idx_with_commit!(commit, latest: latest)
2765
2674
  @project.generate_idx
2675
+
2766
2676
  log_message("Jumped successfuly!", Thor::Shell::Color::GREEN)
2767
2677
  rescue => e
2678
+ Logger::log_error(e)
2768
2679
  log_message("Cant jump to the specified commit", Thor::Shell::Color::RED)
2769
2680
  end
2770
- # successful_changes = []
2771
- #
2772
- # if !response["result"]["tree"].nil?
2773
-
2774
- # commit_sha1 = response["result"]["commit"]
2775
- # idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
2776
- # File.open(project_home + "/.cnvrg/idx.yml", "w+") {|f| f.write idx.to_yaml}
2777
- # if is_remote
2778
- # current_tree = Dir.glob("**/*", File::FNM_DOTMATCH).flatten.reject {|file| file.start_with? '.' or file.eql? "__init__.py" or file.eql? "uwsgi.ini" or file.ends_with? "/." or file.eql? "."}
2779
- # else
2780
- # current_tree = Dir.glob("**/*", File::FNM_DOTMATCH).flatten.reject {|file| file.start_with? '.cnvrg' or file.ends_with? "/." or file.eql? "."}
2781
- # end
2782
- #
2783
- # jump_result = Parallel.map(response["result"]["tree"], parallel_options) do |f|
2784
- #
2785
- # relative_path = f[0].gsub(/^#{@project.local_path}/, "")
2786
- # log_message("Downloading #{f[0]}", Thor::Shell::Color::BLUE, false)
2787
- # if f[0].end_with? "/"
2788
- # # dir
2789
- # @files.download_dir(f[0], relative_path, project_home)
2790
- #
2791
- # else
2792
- # # blob
2793
- # @files.download_file_s3(f[0], relative_path, project_home, commit_sha1 = commit_sha1)
2794
- # end
2795
- # end
2796
- #
2797
- #
2798
- # successful_changes = jump_result.select {|x| not x.nil?}
2799
- # end
2800
- # response["result"]["tree"].each do |f|
2801
- # if f[0].end_with? "/"
2802
- # current_tree.delete(f[0][0, f[0].size - 1])
2803
- # else
2804
- # current_tree.delete(f[0])
2805
- # end
2806
- # end
2807
- #
2808
- # FileUtils.rm_rf(current_tree)
2809
- # log_message("Done. Jumped to #{commit_sha1} completed successfully", Thor::Shell::Color::GREEN)
2810
- # rescue => e
2811
- # log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
2812
- # log_error(e)
2813
- # rescue SignalException
2814
- #
2815
- # exit(1)
2816
- # end
2817
2681
  end
2818
2682
 
2819
2683
  desc 'show', 'Show specific file from a specific commit'
@@ -2911,6 +2775,7 @@ module Cnvrg
2911
2775
  method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
2912
2776
  def sync(direct = true)
2913
2777
  verify_logged_in(true) if direct
2778
+ @project = Project.new(get_project_home)
2914
2779
  log_start(__method__, args, options)
2915
2780
  log_message('Checking for new updates from remote version', Thor::Shell::Color::BLUE, options["verbose"])
2916
2781
  log_message('Syncing project', Thor::Shell::Color::BLUE, !options["verbose"])
@@ -2919,7 +2784,7 @@ module Cnvrg
2919
2784
  in_exp = options["in_exp"] || (job_slug.present? and job_type.present?)
2920
2785
  in_exp = false if job_type.present? and job_type == "NotebookSession"
2921
2786
  run_download = true
2922
- if options[:force] or options[:files].present? or options[:output_dir].present? or in_exp
2787
+ if options[:force] or options[:files].present? or options[:output_dir].present? or in_exp or @project.is_branch
2923
2788
  run_download = false
2924
2789
  end
2925
2790
  if run_download
@@ -3477,10 +3342,6 @@ module Cnvrg
3477
3342
  else
3478
3343
  sync_result = invoke :sync, [false], :force => false
3479
3344
  end
3480
-
3481
-
3482
-
3483
-
3484
3345
  end
3485
3346
  #handle grid if it's git project
3486
3347
  if project.is_git and grid.present?
@@ -3516,6 +3377,7 @@ module Cnvrg
3516
3377
  if forced_commit and (commit_to_run.nil? or commit_to_run.empty?)
3517
3378
  commit_to_run = forced_commit
3518
3379
  end
3380
+
3519
3381
  commit_to_run = commit_to_run.presence || project.last_local_commit
3520
3382
 
3521
3383
  res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
@@ -5491,7 +5353,7 @@ module Cnvrg
5491
5353
  end
5492
5354
  end
5493
5355
 
5494
- def log_message(message, type, to_print = true)
5356
+ def log_message(message, type=Thor::Shell::Color::GREEN, to_print = true)
5495
5357
  if to_print
5496
5358
  say message, type
5497
5359
  end
@@ -5538,10 +5400,10 @@ module Cnvrg
5538
5400
  error = response['message'] || "Unknown error"
5539
5401
  # Cnvrg::CLI.log_end(1, error)
5540
5402
  if response["status"] == 500
5541
- say("<%= color('Server Error', RED) %>")
5403
+ log_message('Server Error', Thor::Shell::Color::RED)
5542
5404
  else
5543
5405
  $LOG.error message: error, type: "error"
5544
- say("<%= color('Error:#{error}', RED) %>")
5406
+ log_message("error", Thor::Shell::Color::RED)
5545
5407
  end
5546
5408
 
5547
5409
  if should_exit
@@ -5555,7 +5417,7 @@ module Cnvrg
5555
5417
  puts e.message
5556
5418
  puts e.backtrace
5557
5419
  rescue SignalException
5558
- say "Aborting"
5420
+ log_message "Aborting", Thor::Shell::Color::RED
5559
5421
  end
5560
5422
 
5561
5423
  end
@@ -5837,7 +5699,7 @@ module Cnvrg
5837
5699
  rescue
5838
5700
  return false
5839
5701
  rescue SignalException
5840
- say "\nAborting"
5702
+ say "\nAborting", Thor::Shell::Color::RED
5841
5703
  Exit(0)
5842
5704
  end
5843
5705
 
@@ -567,17 +567,15 @@ module Cnvrg
567
567
  end
568
568
 
569
569
 
570
- def download_file_s3(absolute_path, relative_path, project_home, commit_sha1=nil, conflict=false)
570
+ def download_file_s3(relative_path, commit_sha1=nil, postfix: '')
571
571
  begin
572
- res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path,
572
+ res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {relative_path: relative_path,
573
573
  commit_sha1: commit_sha1,new_version:true})
574
574
 
575
575
  Cnvrg::CLI.is_response_success(res, false)
576
576
  if res["result"]
577
577
  download_resp = res
578
578
  filename = download_resp["result"]["filename"]
579
-
580
- absolute_path += ".conflict" if conflict
581
579
  sts_path = download_resp["result"]["path_sts"]
582
580
  retries = 0
583
581
  success= false
@@ -632,8 +630,8 @@ module Cnvrg
632
630
  :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
633
631
  :http_open_timeout => 60, :retry_limit => 20)
634
632
  end
635
-
636
- File.open(project_home+"/"+absolute_path, 'w+') do |file|
633
+ absolute_path = "#{@project_home}/#{relative_path}#{postfix}"
634
+ File.open(absolute_path, 'w+') do |file|
637
635
  resp = client.get_object({bucket:bucket,
638
636
  key:file_key}, target: file)
639
637
  end
@@ -649,7 +647,21 @@ module Cnvrg
649
647
  end
650
648
  end
651
649
 
650
+ def create_progressbar(length = 10, title = 'Progress')
651
+ ProgressBar.create(:title => title,
652
+ :progress_mark => '=',
653
+ :format => "%b>>%i| %p%% %t",
654
+ :starting_at => 0,
655
+ :total => length,
656
+ :autofinish => true)
657
+ end
658
+
652
659
  def download_files(files, commit, postfix: '', progress: nil)
660
+ return if files.blank?
661
+ if Cnvrg::Helpers.server_version < 1
662
+ Cnvrg::Logger.log_info("Download files from older server.")
663
+ return self.download_files_old(files, commit, progress: progress, postfix: postfix)
664
+ end
653
665
  res = Cnvrg::API.request(@base_resource + "download_files", 'POST', {files: files, commit: commit})
654
666
  unless Cnvrg::CLI.is_response_success(res, false)
655
667
  raise Exception.new("Cant download files from the server.")
@@ -657,6 +669,14 @@ module Cnvrg
657
669
  self.download_multpile_files_s3(res['result'], @project_home, postfix: postfix, progress: progress)
658
670
  end
659
671
 
672
+
673
+ def download_files_old(files, commit, postfix: '', progress: nil)
674
+ files.each do |file|
675
+ self.download_file_s3(file, commit, postfix: postfix)
676
+ progress.progress += 1 if progress.present?
677
+ end
678
+ end
679
+
660
680
  def delete_files_local(deleted, conflicted: [], progress: nil)
661
681
  deleted -= conflicted
662
682
  deleted.each{|file| self.delete(file); progress.progress += 1 if progress.present?}
@@ -11,7 +11,7 @@ module Cnvrg
11
11
  begin
12
12
  @local_path = project_home
13
13
  @working_dir = project_home
14
- config = YAML.load_file(project_home+"/.cnvrg/config.yml")
14
+ config = YAML.load_file(project_home + "/.cnvrg/config.yml")
15
15
  @title = config[:project_name]
16
16
  @slug = config[:project_slug]
17
17
  @owner = config[:owner]
@@ -45,7 +45,7 @@ module Cnvrg
45
45
  end
46
46
  list = new_ignore.split(",")
47
47
  begin
48
- File.open(self.local_path+"/.cnvrgignore", "a+") do |f|
48
+ File.open(self.local_path + "/.cnvrgignore", "a+") do |f|
49
49
  f.puts("\n")
50
50
  list.each do |i|
51
51
  f.puts("#{i}\n")
@@ -59,16 +59,16 @@ module Cnvrg
59
59
 
60
60
  def get_ignore_list
61
61
  ignore_list = []
62
- if !File.exist? self.local_path+"/.cnvrgignore"
62
+ if !File.exist? self.local_path + "/.cnvrgignore"
63
63
  return ignore_list
64
64
  end
65
- File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
65
+ File.open(self.local_path + "/.cnvrgignore", "r").each_line do |line|
66
66
  line = line.strip
67
67
  if line.start_with? "#" or ignore_list.include? line or line.empty?
68
68
  next
69
69
  end
70
70
  if line.ends_with? "*"
71
- list_regex = Dir.glob("**/#{line}",File::FNM_DOTMATCH).flatten
71
+ list_regex = Dir.glob("**/#{line}", File::FNM_DOTMATCH).flatten
72
72
  list_regex.each do |l|
73
73
  ignore_list << l
74
74
  if File.directory?(l)
@@ -79,15 +79,15 @@ module Cnvrg
79
79
 
80
80
  end
81
81
  elsif line.ends_with? "/*"
82
- line = line.gsub("/*","")
82
+ line = line.gsub("/*", "")
83
83
  regex_list = Dir.glob("**/#{line}/**/*", File::FNM_DOTMATCH).flatten
84
84
  ignore_list << regex_list
85
85
  elsif line.include? "*"
86
- regex_list = Dir.glob("**/#{line}").flatten
87
- ignore_list << regex_list
86
+ regex_list = Dir.glob("**/#{line}").flatten
87
+ ignore_list << regex_list
88
88
  elsif line.end_with? "/" or File.directory?(line)
89
- ignore_list << line
90
- all_sub = Dir.glob("#{line}/**/*", File::FNM_DOTMATCH).flatten
89
+ ignore_list << line
90
+ all_sub = Dir.glob("#{line}/**/*", File::FNM_DOTMATCH).flatten
91
91
 
92
92
  ignore_list << all_sub.flatten
93
93
 
@@ -102,16 +102,16 @@ module Cnvrg
102
102
  def send_ignore_list()
103
103
  begin
104
104
  ignore_list = []
105
- File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
105
+ File.open(self.local_path + "/.cnvrgignore", "r").each_line do |line|
106
106
  line = line.strip
107
107
  if line.start_with? "#" or ignore_list.include? line or line.empty?
108
108
  next
109
109
  end
110
110
  if line.end_with? "/"
111
- ignore_list << line.gsub("/","")
112
- ignore_list << line+"."
111
+ ignore_list << line.gsub("/", "")
112
+ ignore_list << line + "."
113
113
  elsif line.include? "*"
114
- line = line.gsub("*",".*")
114
+ line = line.gsub("*", ".*")
115
115
  ignore_list << line
116
116
  else
117
117
  ignore_list << line
@@ -126,7 +126,7 @@ module Cnvrg
126
126
 
127
127
  # Create project
128
128
 
129
- def self.create(project_name, clean, with_docker=false)
129
+ def self.create(project_name, clean, with_docker = false)
130
130
  if clean
131
131
  list_dirs = [project_name, project_name + "/.cnvrg"]
132
132
  else
@@ -149,7 +149,7 @@ module Cnvrg
149
149
  ]
150
150
  cnvrgreadme = Helpers.readme_content
151
151
  cnvrgignore = Helpers.cnvrgignore_content
152
- cnvrghyper = Helpers.hyper_content
152
+ cnvrghyper = Helpers.hyper_content
153
153
 
154
154
  begin
155
155
 
@@ -166,10 +166,10 @@ module Cnvrg
166
166
  FileUtils.mkdir_p list_dirs
167
167
  FileUtils.touch list_files
168
168
 
169
- File.open(project_name + "/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
170
- File.open(project_name + "/.cnvrgignore", "w+") { |f| f.write cnvrgignore }
171
- File.open(project_name + "/README.md", "w+") { |f| f.write cnvrgreadme }
172
- File.open(project_name + "/src/hyper.yaml", "w+") { |f| f.write cnvrghyper }
169
+ File.open(project_name + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
170
+ File.open(project_name + "/.cnvrgignore", "w+") {|f| f.write cnvrgignore}
171
+ File.open(project_name + "/README.md", "w+") {|f| f.write cnvrgreadme}
172
+ File.open(project_name + "/src/hyper.yaml", "w+") {|f| f.write cnvrghyper}
173
173
 
174
174
  rescue
175
175
  return false
@@ -177,75 +177,76 @@ module Cnvrg
177
177
  return true
178
178
  end
179
179
 
180
- def self.link(owner, project_name, docker=false, git = false)
181
- ignore_exits = File.exist? ".cnvrgignore"
182
- list_dirs = [".cnvrg"
183
- ]
184
- list_files = [
185
- ".cnvrg/config.yml"
186
- ]
187
- if !ignore_exits
188
- list_files <<
189
- ".cnvrgignore"
190
- end
180
+ def self.link(owner, project_name, docker = false, git = false)
181
+ ignore_exits = File.exist? ".cnvrgignore"
182
+ list_dirs = [".cnvrg"
183
+ ]
184
+ list_files = [
185
+ ".cnvrg/config.yml"
186
+ ]
187
+ if !ignore_exits
188
+ list_files <<
189
+ ".cnvrgignore"
190
+ end
191
191
 
192
- cnvrgreadme = Helpers.readme_content
193
- cnvrgignore = Helpers.cnvrgignore_content
194
- begin
195
- response = Cnvrg::API.request("cli/create_project", 'POST', {title: project_name, owner: owner, is_docker: docker})
196
- Cnvrg::CLI.is_response_success(response)
197
- response = JSON.parse response["result"]
198
- project_slug = response["slug"]
199
-
200
- config = {project_name: project_name,
201
- project_slug: project_slug,
202
- owner: owner,
203
- git: git}
204
- FileUtils.mkdir_p list_dirs
205
- FileUtils.touch list_files
206
- File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
207
- File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless ignore_exits
208
- if !File.exist? "README" and !File.exist? "README.md"
209
- FileUtils.touch [ "README.md" ]
210
- File.open("README.md", "w+") { |f| f.write cnvrgreadme }
211
- end
192
+ cnvrgreadme = Helpers.readme_content
193
+ cnvrgignore = Helpers.cnvrgignore_content
194
+ begin
195
+ response = Cnvrg::API.request("cli/create_project", 'POST', {title: project_name, owner: owner, is_docker: docker})
196
+ Cnvrg::CLI.is_response_success(response)
197
+ response = JSON.parse response["result"]
198
+ project_slug = response["slug"]
212
199
 
213
- rescue => e
214
- puts e
215
- return false
200
+ config = {project_name: project_name,
201
+ project_slug: project_slug,
202
+ owner: owner,
203
+ git: git}
204
+ FileUtils.mkdir_p list_dirs
205
+ FileUtils.touch list_files
206
+ File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
207
+ File.open(".cnvrgignore", "w+") {|f| f.write cnvrgignore} unless ignore_exits
208
+ if !File.exist? "README" and !File.exist? "README.md"
209
+ FileUtils.touch ["README.md"]
210
+ File.open("README.md", "w+") {|f| f.write cnvrgreadme}
216
211
  end
217
- return true
212
+
213
+ rescue => e
214
+ puts e
215
+ return false
218
216
  end
217
+ return true
218
+ end
219
219
 
220
- def self.clone_dir(project_slug, project_owner, project_name,is_git=false)
221
- list_dirs = [project_name,
222
- project_name + "/.cnvrg"
223
- ]
220
+ def self.clone_dir(project_slug, project_owner, project_name, is_git = false)
221
+ list_dirs = [project_name,
222
+ project_name + "/.cnvrg"
223
+ ]
224
224
 
225
225
 
226
- list_files = [
227
- project_name + "/.cnvrg/config.yml",
228
- project_name+"/.cnvrgignore",
229
- ]
230
- begin
231
- config = {project_name: project_name,
232
- project_slug: project_slug,
233
- owner: project_owner,
234
- git: is_git}
235
- FileUtils.mkdir_p list_dirs
236
- FileUtils.touch list_files
237
- cnvrgignore = Helpers.cnvrgignore_content
226
+ list_files = [
227
+ project_name + "/.cnvrg/config.yml",
228
+ project_name + "/.cnvrgignore",
229
+ ]
230
+ begin
231
+ config = {project_name: project_name,
232
+ project_slug: project_slug,
233
+ owner: project_owner,
234
+ git: is_git}
235
+ FileUtils.mkdir_p list_dirs
236
+ FileUtils.touch list_files
237
+ cnvrgignore = Helpers.cnvrgignore_content
238
238
 
239
239
 
240
- File.open(project_name + "/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
241
- File.open(project_name+"/.cnvrgignore", "w+") { |f| f.write cnvrgignore }
240
+ File.open(project_name + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
241
+ File.open(project_name + "/.cnvrgignore", "w+") {|f| f.write cnvrgignore}
242
242
 
243
- rescue
244
- return false
245
- end
246
- return true
243
+ rescue
244
+ return false
247
245
  end
248
- def self.verify_cnvrgignore_exist(project_name,remote)
246
+ return true
247
+ end
248
+
249
+ def self.verify_cnvrgignore_exist(project_name, remote)
249
250
  path = ".cnvrgignore"
250
251
  if !File.exist? path
251
252
  path = "#{project_name}/.cnvrgignore"
@@ -255,10 +256,10 @@ module Cnvrg
255
256
  begin
256
257
  list_files = [
257
258
  path
258
- ]
259
- FileUtils.touch list_files
260
- cnvrgignore = Helpers.cnvrgignore_content
261
- File.open(path, "w+") { |f| f.write cnvrgignore }
259
+ ]
260
+ FileUtils.touch list_files
261
+ cnvrgignore = Helpers.cnvrgignore_content
262
+ File.open(path, "w+") {|f| f.write cnvrgignore}
262
263
  rescue => e
263
264
  return false
264
265
  end
@@ -266,64 +267,66 @@ module Cnvrg
266
267
  end
267
268
  end
268
269
 
269
- def self.clone_dir_remote(project_slug, project_owner, project_name,is_git=false)
270
- list_dirs = [
271
- ".cnvrg"
272
- ]
273
-
274
-
270
+ def self.clone_dir_remote(project_slug, project_owner, project_name, is_git = false)
271
+ list_dirs = [
272
+ ".cnvrg"
273
+ ]
275
274
 
276
- list_files = [
277
- ".cnvrg/config.yml",
278
275
 
279
- ]
276
+ list_files = [
277
+ ".cnvrg/config.yml",
280
278
 
279
+ ]
281
280
 
282
- begin
283
- config = {project_name: project_name,
284
- project_slug: project_slug,
285
- owner: project_owner,
286
- git: is_git
287
- }
288
- FileUtils.mkdir_p list_dirs
289
- FileUtils.touch list_files
290
281
 
282
+ begin
283
+ config = {project_name: project_name,
284
+ project_slug: project_slug,
285
+ owner: project_owner,
286
+ git: is_git
287
+ }
288
+ FileUtils.mkdir_p list_dirs
289
+ FileUtils.touch list_files
291
290
 
292
- File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
293
- if !File.exist? ".cnvrgignore"
294
- FileUtils.touch ".cnvrgignore"
295
- list_files << ".cnvrgignore"
296
- cnvrgignore = Helpers.cnvrgignore_content
297
- File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore }
298
291
 
292
+ File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
293
+ if !File.exist? ".cnvrgignore"
294
+ FileUtils.touch ".cnvrgignore"
295
+ list_files << ".cnvrgignore"
296
+ cnvrgignore = Helpers.cnvrgignore_content
297
+ File.open(".cnvrgignore", "w+") {|f| f.write cnvrgignore}
299
298
 
300
- end
301
299
 
302
- rescue
303
- return false
304
300
  end
305
- return true
301
+
302
+ rescue
303
+ return false
306
304
  end
305
+ return true
306
+ end
307
+
307
308
  def update_is_new_branch(new_branch)
308
- config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
309
+ config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
309
310
  config[:new_branch] = new_branch
310
- File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
311
+ File.open(@working_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
311
312
  end
313
+
312
314
  def get_new_branch
313
315
  begin
314
- config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
315
- return config[:new_branch]
316
- rescue =>e
316
+ config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
317
+ return config[:new_branch]
318
+ rescue => e
317
319
  return false
318
320
 
319
321
  end
320
322
  end
321
323
 
322
324
  def remove_new_branch
323
- config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
325
+ config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
324
326
  new_config = config.except(:new_branch)
325
- File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write new_config.to_yaml }
327
+ File.open(@working_dir + "/.cnvrg/config.yml", "w+") {|f| f.write new_config.to_yaml}
326
328
  end
329
+
327
330
  def generate_output_dir(output_dir)
328
331
  Cnvrg::Logger.log_info("Generating output dir for #{output_dir}")
329
332
  upload_list = []
@@ -332,10 +335,10 @@ module Cnvrg
332
335
  next if e.end_with? "/."
333
336
  if File.directory? e
334
337
 
335
- upload_list << e+"/"
338
+ upload_list << e + "/"
336
339
  else
337
340
  upload_list << e
338
- end
341
+ end
339
342
  end
340
343
  if Dir.exists? output_dir
341
344
  upload_list << output_dir + "/"
@@ -364,130 +367,132 @@ module Cnvrg
364
367
 
365
368
  end
366
369
 
367
- def generate_idx(deploy:false)
368
- if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
369
- old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
370
- else
371
- old_idx = {:tree => {}, :commit => nil}
372
- end
370
+ def generate_idx(deploy: false)
371
+ if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
372
+ old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
373
+ else
374
+ old_idx = {:tree => {}, :commit => nil}
375
+ end
373
376
 
374
- tree_idx = Hash.new(0)
375
- list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~ /^#{self.local_path}\/\.git\/*/) or (x =~/^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~/^#{self.local_path}\/\.cnvrgignore/) }
376
- list_ignore = self.get_ignore_list()
377
- if deploy
378
- list_ignore << ["main.py","main.pyc", "__init__.py", "uwsgi.ini"]
379
- list_ignore.flatten!
380
- end
377
+ tree_idx = Hash.new(0)
378
+ list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject {|x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~ /^#{self.local_path}\/\.git\/*/) or (x =~ /^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~ /^#{self.local_path}\/\.cnvrgignore/)}
379
+ list_ignore = self.get_ignore_list()
380
+ if deploy
381
+ list_ignore << ["main.py", "main.pyc", "__init__.py", "uwsgi.ini"]
382
+ list_ignore.flatten!
383
+ end
381
384
 
382
- Parallel.map(list, in_threads: IDXParallelThreads) do |e|
383
- label = e.gsub(self.local_path + "/", "")
384
- ignore_label = label.gsub("/","//")
385
- if list_ignore.include? ignore_label
386
- next
387
- end
388
- if File.directory? e
385
+ Parallel.map(list, in_threads: IDXParallelThreads) do |e|
386
+ label = e.gsub(self.local_path + "/", "")
387
+ ignore_label = label.gsub("/", "//")
388
+ if list_ignore.include? ignore_label
389
+ next
390
+ end
391
+ if File.directory? e
389
392
 
390
- tree_idx[label+"/"] = nil
393
+ tree_idx[label + "/"] = nil
394
+ else
395
+ sha1 = OpenSSL::Digest::SHA1.file(e).hexdigest
396
+ if old_idx.nil? or old_idx.to_h[:tree].nil?
397
+ tree_idx[label] = {sha1: sha1, commit_time: nil}
398
+ elsif old_idx[:tree][label].nil? or old_idx[:tree][label][:sha1] != sha1
399
+ tree_idx[label] = {sha1: sha1, commit_time: nil}
391
400
  else
392
- sha1 = OpenSSL::Digest::SHA1.file(e).hexdigest
393
- if old_idx.nil? or old_idx.to_h[:tree].nil?
394
- tree_idx[label] = {sha1: sha1, commit_time: nil}
395
- elsif old_idx[:tree][label].nil? or old_idx[:tree][label][:sha1] != sha1
396
- tree_idx[label] = {sha1: sha1, commit_time: nil}
397
- else
398
- tree_idx[label] = old_idx[:tree][label]
399
- end
401
+ tree_idx[label] = old_idx[:tree][label]
400
402
  end
401
403
  end
404
+ end
402
405
 
403
- old_idx[:tree] = tree_idx
406
+ old_idx[:tree] = tree_idx
404
407
 
405
- File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write old_idx.to_yaml }
406
- return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
407
- end
408
- def get_idx
409
- unless File.exists? "#{self.local_path}/.cnvrg/idx.yml"
410
- empty_idx = {:commit => nil, :tree => {}}
411
- File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write empty_idx.to_yaml }
412
- return empty_idx
413
- end
414
- YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
408
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write old_idx.to_yaml}
409
+ return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
410
+ end
411
+
412
+ def get_idx
413
+ unless File.exists? "#{self.local_path}/.cnvrg/idx.yml"
414
+ empty_idx = {:commit => nil, :tree => {}}
415
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write empty_idx.to_yaml}
416
+ return empty_idx
415
417
  end
418
+ YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
419
+ end
416
420
 
417
421
  def set_idx(idx)
418
422
  FileUtils.mkdir_p("#{self.local_path}/.cnvrg")
419
- File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
423
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx.to_yaml}
424
+ end
425
+
426
+ def clone(remote = 0, commit)
427
+ response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/clone", 'POST', {project_slug: self.slug, remote: remote, commit: commit})
428
+ return response
420
429
  end
421
430
 
422
- def clone(remote=0, commit)
423
- response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/clone", 'POST', {project_slug: self.slug, remote: remote, commit: commit})
424
- return response
425
- end
426
431
  def git_download_commit(commit)
427
432
  response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/git_download_commit", 'POST', {commit_sha1: commit})
428
- CLI.is_response_success(response,true)
433
+ CLI.is_response_success(response, true)
429
434
  return response
430
435
  end
431
436
 
432
437
  def compare_idx(new_branch, force:false, deploy: false, in_exp:false, specific_files: [], download: false)
433
- is_download = download
434
- if is_download
435
- local_idx = self.get_idx
436
- else
437
- #upload
438
- local_idx = self.generate_idx(deploy: deploy)
439
- end
440
- commit = local_idx[:commit]
441
- tree = local_idx[:tree]
442
- ignore_list = self.send_ignore_list()
443
- if force or specific_files.present?
444
- added = []
445
- if specific_files.present?
446
- added = specific_files
447
- elsif tree.present?
448
- added << local_idx[:tree].keys
449
- added.flatten!
450
- end
451
- response ={"result"=> {"commit"=>nil,"tree"=> {"added"=> added,
452
- "updated_on_server"=> [],
453
- "updated_on_local"=> [],
454
- "update_local" => [],
455
- "deleted"=> [],
456
- "conflicts"=> []} } }
457
- return response
438
+ is_download = download
439
+ if is_download
440
+ local_idx = self.get_idx
441
+ else
442
+ #upload
443
+ local_idx = self.generate_idx(deploy: deploy)
444
+ end
445
+ commit = local_idx[:commit]
446
+ tree = local_idx[:tree]
447
+ ignore_list = self.send_ignore_list()
448
+ if force or specific_files.present?
449
+ added = []
450
+ if specific_files.present?
451
+ added = specific_files
452
+ elsif tree.present?
453
+ added << local_idx[:tree].keys
454
+ added.flatten!
458
455
  end
459
- #we dont want to send it on download - we only compare between commits sha1 in download.
460
- if is_download
461
- #the new server doesnt need the tree, but the old probably needs :X
462
- local_idx[:tree] = {} if Cnvrg::Helpers.server_version > 0
456
+ response = {"result" => {"commit" => nil, "tree" => {"added" => added,
457
+ "updated_on_server" => [],
458
+ "updated_on_local" => [],
459
+ "update_local" => [],
460
+ "deleted" => [],
461
+ "conflicts" => []}}}
462
+ return response
463
+ end
464
+ #we dont want to send it on download - we only compare between commits sha1 in download.
465
+ if is_download
466
+ #the new server doesnt need the tree, but the old probably needs :X
467
+ local_idx[:tree] = {} if Cnvrg::Helpers.server_version > 0
468
+ end
469
+ response = Cnvrg::API.request(@base_resource + "status", 'POST', {idx: local_idx, new_branch: new_branch,
470
+ current_commit: commit, ignore: ignore_list, force: force, in_exp: in_exp, download: download})
471
+
472
+ CLI.is_response_success(response, true)
473
+ if is_download
474
+ if Cnvrg::Helpers.server_version > 0
475
+ #trying to optimize the download using resolver
476
+ resolve = response['result']['tree']['resolver'] || tree #tree of file -> sha1 from current commit to check conflicts
477
+ destination_files = response['result']['tree']['destination'] || {} #tree of file -> sha1 from last commit to check files that already downloaded
478
+ else
479
+ resolve = tree
480
+ destination_files = {}
463
481
  end
464
- response = Cnvrg::API.request(@base_resource + "status", 'POST', {idx: local_idx, new_branch: new_branch,
465
- current_commit: commit,ignore:ignore_list, force:force,in_exp:in_exp, download: download})
466
-
467
- CLI.is_response_success(response,true)
468
- if is_download
469
- if Cnvrg::Helpers.server_version > 0
470
- #trying to optimize the download using resolver
471
- resolve = response['result']['tree']['resolver'] || tree #tree of file -> sha1 from current commit to check conflicts
472
- destination_files = response['result']['tree']['destination'] || {} #tree of file -> sha1 from last commit to check files that already downloaded
473
- else
474
- resolve = tree
475
- destination_files = {}
476
- end
477
- @files = self.get_files
478
- local_tree = @files.calculate_sha1(resolve.keys)
479
- changed_files = resolve.keys.select{|file| local_tree[file] != resolve[file]}
482
+ @files = self.get_files
483
+ local_tree = @files.calculate_sha1(resolve.keys)
484
+ changed_files = resolve.keys.select {|file| local_tree[file] != resolve[file]}
480
485
 
481
- # means that the user changed the file locally
482
- response['result']['tree']['update_local'] = changed_files
486
+ # means that the user changed the file locally
487
+ response['result']['tree']['update_local'] = changed_files
483
488
 
484
- # means that we already downloaded this file and we dont need it anymore
485
- downloaded_files = destination_files.keys.select{|file| local_tree[file] == destination_files[file]}
486
- response['result']['tree']['added'] -= downloaded_files
487
- response['result']['tree']['updated_on_server'] -= downloaded_files
488
- end
489
- return response
489
+ # means that we already downloaded this file and we dont need it anymore
490
+ downloaded_files = destination_files.keys.select {|file| local_tree[file] == destination_files[file]}
491
+ response['result']['tree']['added'] -= downloaded_files
492
+ response['result']['tree']['updated_on_server'] -= downloaded_files
490
493
  end
494
+ return response
495
+ end
491
496
 
492
497
  def get_files
493
498
  Cnvrg::Files.new(self.owner, self.slug, project_home: @local_path)
@@ -502,52 +507,52 @@ module Cnvrg
502
507
  'POST',
503
508
  {tree: tree, ignore: ignore_list,
504
509
  dest_commit: destination, current_commit: current_commit})
505
- CLI.is_response_success(response,false)
510
+ CLI.is_response_success(response, false)
506
511
  response
507
512
  end
508
513
 
509
514
  def set_on_branch(is_latest)
510
- config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
515
+ config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
511
516
  @is_branch = !is_latest
512
517
  config[:is_branch] = @is_branch
513
518
 
514
- File.open(@working_dir+"/.cnvrg/config.yml", 'w') { |f| f.write config.to_yaml }
519
+ File.open(@working_dir + "/.cnvrg/config.yml", 'w') {|f| f.write config.to_yaml}
515
520
 
516
521
  end
517
522
 
518
- def compare_commit(commit)
519
- if commit.nil? or commit.empty?
520
- commit = last_local_commit
521
- end
522
- response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
523
- CLI.is_response_success(response,false)
524
- update_is_new_branch(response["result"]["new_branch"])
525
- return response["result"]["new_branch"]
523
+ def compare_commit(commit)
524
+ if commit.nil? or commit.empty?
525
+ commit = last_local_commit
526
526
  end
527
+ response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
528
+ CLI.is_response_success(response, false)
529
+ update_is_new_branch(response["result"]["new_branch"])
530
+ return response["result"]["new_branch"]
531
+ end
527
532
 
528
- def update_idx_with_files_commits!(files, commit_time)
529
-
530
- idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
531
- files.each do |path|
532
- idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
533
- end
534
- File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
533
+ def update_idx_with_files_commits!(files, commit_time)
535
534
 
536
- return true
535
+ idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
536
+ files.each do |path|
537
+ idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
537
538
  end
539
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
540
+
541
+ return true
542
+ end
538
543
 
539
- def deploy(file_to_run, function, input_params, commit_to_run, instance_type, image_slug, scheduling_query, local_timestamp,workers, file_input, title)
544
+ def deploy(file_to_run, function, input_params, commit_to_run, instance_type, image_slug, scheduling_query, local_timestamp, workers, file_input, title)
545
+ response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/deploy", 'POST', {file_to_run: file_to_run, function: function,
546
+ image_slug: image_slug, input_params: input_params,
547
+ commit_sha1: commit_to_run,
548
+ instance_type: instance_type,
549
+ scheduling_query: scheduling_query,
550
+ local_timestamp: local_timestamp,
551
+ workers: workers, file_input: file_input,
552
+ title: title})
553
+ return response
554
+ end
540
555
 
541
- response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/deploy", 'POST', {file_to_run: file_to_run, function: function,
542
- image_slug: image_slug, input_params: input_params,
543
- commit_sha1: commit_to_run,
544
- instance_type: instance_type,
545
- scheduling_query: scheduling_query,
546
- local_timestamp: local_timestamp,
547
- workers:workers,file_input:file_input,
548
- title: title})
549
- return response
550
- end
551
556
  def list_commits
552
557
  response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commits/list", 'GET')
553
558
  CLI.is_response_success(response)
@@ -567,17 +572,19 @@ module Cnvrg
567
572
  CLI.is_response_success(response)
568
573
  return response
569
574
  end
570
- def update_idx_with_commit!(commit)
571
- idx_hash = self.get_idx
572
- idx_hash[:commit] = commit
573
575
 
574
- File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
575
- return true
576
- end
576
+ def update_idx_with_commit!(commit, latest: nil)
577
+ idx_hash = self.get_idx
578
+ idx_hash[:commit] = commit
579
+ self.set_on_branch(latest) unless latest.nil?
577
580
 
578
- def revert(working_dir)
579
- FileUtils.rm_rf working_dir
580
- end
581
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
582
+ return true
583
+ end
584
+
585
+ def revert(working_dir)
586
+ FileUtils.rm_rf working_dir
587
+ end
581
588
 
582
589
  def init_machines
583
590
  Cnvrg::Logger.log_info("Init machines.")
@@ -599,5 +606,6 @@ module Cnvrg
599
606
  machines.include? machine
600
607
  end
601
608
 
602
- end
609
+
603
610
  end
611
+ end
@@ -1,4 +1,4 @@
1
1
  module Cnvrg
2
- VERSION = '0.8.5'
2
+ VERSION = '0.8.6'
3
3
  end
4
4
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cnvrg
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.5
4
+ version: 0.8.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yochay Ettun