cnvrg 0.8.5 → 0.8.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: cb82613324f1a1f409c0819b795be4a580bf6f31
4
- data.tar.gz: fb950e5920d8cae1cd10609ec4ba0eba1707bfbf
3
+ metadata.gz: 819730717280428ac1ba231d43462cb827253cc1
4
+ data.tar.gz: 95c1963b178296af0655d54b691b02895ced41ef
5
5
  SHA512:
6
- metadata.gz: 3d51178739e2851c1871de672d85ca1c69d099cdb9f94f3b80a222ae1cc30cbfae60a9d1892995be0657c9cf5f8eb406cbd9ae30cda3a4183891a87370f226d0
7
- data.tar.gz: 3cb42c96a9e8a04e7a0ee672e5ac404ae46b4668b96e6edc107ee89d46a48e64764f5b46cabfb163e060b00b3982fb5a7a7ac7755d56d9aaf72bb0b9e8a9e01f
6
+ metadata.gz: 61563035b65f94b7d87354a6dcb783119179edf655cd6881595035417d15c6441b0337f48ae9de9b410136bd3133eb3254fb995af46ffbd67d785e09312aa3e2
7
+ data.tar.gz: fda3bfcd38199e6c04c46a2c2ca7812182e58352c0a0c9d06898acca3f5de30178160d35ab6b6d86d640d7c89ecb1a773a562d44af369e652b5e476943aa8c21
@@ -305,7 +305,7 @@ module Cnvrg
305
305
  compression_path = "#{home_dir}/.cnvrg/tmp"
306
306
  config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url, compression_path: compression_path}
307
307
  File.open(home_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
308
- say "Done"
308
+ say "Done", Thor::Shell::Color::GREEN
309
309
  rescue
310
310
  say "ERROR", Thor::Shell::Color::RED
311
311
  File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
@@ -340,7 +340,7 @@ module Cnvrg
340
340
  compression_path = "#{home_dir}/.cnvrg/tmp"
341
341
  config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url, compression_path: compression_path}
342
342
  File.open(home_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
343
- say "Done"
343
+ say "Done", Thor::Shell::Color::GREEN
344
344
  rescue
345
345
  say "ERROR", Thor::Shell::Color::RED
346
346
  end
@@ -408,7 +408,7 @@ module Cnvrg
408
408
  end
409
409
  end
410
410
  rescue SignalException
411
- say "\nAborting"
411
+ say "\nAborting", Thor::Shell::Color::RED
412
412
  exit(1)
413
413
  end
414
414
  end
@@ -442,7 +442,7 @@ module Cnvrg
442
442
  say "#{checks} Done", Thor::Shell::Color::GREEN
443
443
 
444
444
  rescue SignalException
445
- say "\nAborting"
445
+ say "\nAborting", Thor::Shell::Color::RED
446
446
  exit(1)
447
447
  end
448
448
  end
@@ -544,7 +544,7 @@ module Cnvrg
544
544
  exit(1)
545
545
  rescue SignalException
546
546
 
547
- say "\nAborting"
547
+ say "\nAborting",Thor::Shell::Color::RED
548
548
  logout()
549
549
  exit(1)
550
550
  end
@@ -565,7 +565,7 @@ module Cnvrg
565
565
  puts e.message
566
566
  puts e.backtrace
567
567
  rescue SignalException
568
- say "\nAborting"
568
+ say "\nAborting",Thor::Shell::Color::RED
569
569
  exit(1)
570
570
  end
571
571
 
@@ -587,7 +587,7 @@ module Cnvrg
587
587
 
588
588
 
589
589
  rescue SignalException
590
- say "\nAborting"
590
+ say "\nAborting", Thor::Shell::Color::RED
591
591
  exit(1)
592
592
  end
593
593
  end
@@ -676,7 +676,7 @@ module Cnvrg
676
676
 
677
677
  @project.revert(working_dir)
678
678
  end
679
- say "\nAborting"
679
+ say "\nAborting", Thor::Shell::Color::RED
680
680
  exit(1)
681
681
  end
682
682
  end
@@ -762,7 +762,7 @@ module Cnvrg
762
762
  log_error(e)
763
763
  rescue SignalException
764
764
 
765
- say "\nAborting"
765
+ say "\nAborting", Thor::Shell::Color::RED
766
766
  exit(1)
767
767
  end
768
768
  end
@@ -802,7 +802,7 @@ module Cnvrg
802
802
  rescue => e
803
803
  log_error(e)
804
804
  rescue SignalException
805
- say "\nAborting"
805
+ say "\nAborting", Thor::Shell::Color::RED
806
806
  exit(1)
807
807
  end
808
808
  end
@@ -826,7 +826,7 @@ module Cnvrg
826
826
  log_error(e)
827
827
  rescue SignalException
828
828
 
829
- say "\nAborting"
829
+ say "\nAborting", Thor::Shell::Color::RED
830
830
  exit(1)
831
831
  end
832
832
  end
@@ -891,6 +891,7 @@ module Cnvrg
891
891
  downloaded_files += files['keys'].length
892
892
  files = @files.get_clone_chunk(commit: commit, latest_id: files['latest'])
893
893
  end
894
+ progressbar.finish
894
895
  if downloaded_files == files_count
895
896
  Dataset.verify_cnvrgignore_exist(dataset_name, false)
896
897
  @dataset = Dataset.new(dataset_home)
@@ -905,7 +906,7 @@ module Cnvrg
905
906
  exit(1)
906
907
  end
907
908
  rescue SignalException
908
- say "\nAborting"
909
+ say "\nAborting", Thor::Shell::Color::RED
909
910
  exit(1)
910
911
  end
911
912
  end
@@ -983,7 +984,7 @@ module Cnvrg
983
984
  end
984
985
  end
985
986
  rescue SignalException
986
- say "\nAborting"
987
+ say "\nAborting", Thor::Shell::Color::RED
987
988
  exit(1)
988
989
  end
989
990
  end
@@ -1009,7 +1010,7 @@ module Cnvrg
1009
1010
 
1010
1011
  rescue SignalException
1011
1012
 
1012
- say "\nAborting"
1013
+ say "\nAborting", Thor::Shell::Color::RED
1013
1014
  exit(1)
1014
1015
  end
1015
1016
  end
@@ -1031,7 +1032,7 @@ module Cnvrg
1031
1032
  rescue SignalException
1032
1033
  log_end(-1)
1033
1034
 
1034
- say "\nAborting"
1035
+ say "\nAborting", Thor::Shell::Color::RED
1035
1036
  exit(1)
1036
1037
  end
1037
1038
  end
@@ -1813,112 +1814,22 @@ module Cnvrg
1813
1814
 
1814
1815
  if clone_resp
1815
1816
  @project = Project.new(project_home)
1816
- @files = Cnvrg::Files.new(@project.owner, slug)
1817
+ @files = Cnvrg::Files.new(@project.owner, slug, project_home: project_home)
1817
1818
  response = @project.clone(remote, commit_to_clone)
1818
1819
  Cnvrg::CLI.is_response_success response
1820
+ commit_sha1 = response["result"]["commit"]
1821
+ files = response["result"]["tree"].keys
1819
1822
  idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
1820
-
1821
1823
  log_message("Downloading files", Thor::Shell::Color::BLUE)
1822
- files = response["result"]["files"]
1823
-
1824
- if !files.nil?
1825
- begin
1826
- download_result = @files.download_multpile_files_s3(files, project_home)
1827
- @project.set_idx(idx)
1828
-
1829
- rescue Interrupt
1830
- log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
1831
-
1832
- @files.revoke_clone(project_home)
1833
- return
1834
- end
1835
- Project.verify_cnvrgignore_exist(project_name, remote)
1836
- if !download_result.is_success?
1837
- log_message(download_result.e_msg, download_result.msg_color, false)
1838
- log_message(download_result.e_msg_backtrace, download_result.msg_color, false)
1839
-
1840
- @files.revoke_clone(project_home)
1841
- end
1842
- log_message(download_result.msg, download_result.msg_color)
1843
- else
1844
- current_commit = response["result"]["commit"]
1845
- if !response["result"]["tree"].nil?
1846
- parallel_options = {
1847
- :progress => {
1848
- :title => "Download Progress",
1849
- :progress_mark => '=',
1850
- :format => "%b>>%i| %p%% %t",
1851
- :starting_at => 0,
1852
- :total => response["result"]["tree"].size,
1853
- :autofinish => true
1854
- },
1855
- in_processes: ParallelProcesses,
1856
- in_thread: ParallelThreads
1857
- }
1858
- begin
1859
- successful_changes =[]
1860
- is_success = true
1861
- clone_result = Parallel.map((response["result"]["tree"]), parallel_options) do |f|
1862
-
1863
- relative_path = f[0].gsub(/^#{@project.local_path}/, "")
1864
- if f[0].end_with? "/"
1865
- # dir
1866
- if @files.download_dir(f[0], relative_path, project_home)
1867
- f
1868
- successful_changes << relative_path
1869
- else
1870
- is_success =false
1871
- log_message("Could not create directory: #{f[0]}", Thor::Shell::Color::RED)
1872
- raise Parallel::Kill
1873
- end
1874
- else
1875
- # blob
1876
-
1877
- if @files.download_file_s3(f[0], relative_path, project_home, commit_sha1=current_commit)
1878
- f
1879
- successful_changes << relative_path
1880
- else
1881
- is_success =false
1882
- log_message("Could not download file: #{f[0]}", Thor::Shell::Color::RED)
1883
- raise Parallel::Kill
1884
-
1885
- end
1886
- end
1887
- end
1888
- successful_changes = response["result"]["tree"]
1889
- if !successful_changes.nil? and is_success
1890
- @project.set_idx(idx)
1891
- Project.verify_cnvrgignore_exist(project_name,remote)
1892
- log_message("Done.\nDownloaded #{successful_changes.size}/#{response["result"]["tree"].size} files", Thor::Shell::Color::GREEN)
1893
- else
1894
- log_message("Couldn't download some files", Thor::Shell::Color::RED)
1895
-
1896
- end
1897
- rescue Interrupt
1898
- log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
1899
-
1900
- @files.revoke_clone(project_home)
1901
- return
1902
- end
1903
- end
1904
-
1905
-
1906
- end
1907
-
1908
- else
1909
-
1910
- log_message("Error: Couldn't clone: #{project_name}", Thor::Shell::Color::RED)
1911
- @files.revoke_clone(project_home)
1912
-
1913
- return
1824
+ progressbar = @files.create_progressbar(files.size, "Clone Progress")
1825
+ @files.download_files(files, commit_sha1, progress: progressbar)
1826
+ progressbar.finish
1827
+ Project.verify_cnvrgignore_exist(project_name, remote)
1828
+ @project.set_idx(idx)
1829
+ log_message("Done")
1830
+ log_message("Downloaded #{files.size} files")
1914
1831
  end
1915
- rescue SignalException
1916
- say "\nAborting"
1917
- @files.revoke_clone(project_home)
1918
-
1919
- return
1920
1832
  end
1921
-
1922
1833
  end
1923
1834
 
1924
1835
 
@@ -2360,7 +2271,7 @@ module Cnvrg
2360
2271
  :autofinish => true)
2361
2272
  @files.upload_multiple_files(to_upload, commit_sha1, progress: progressbar)
2362
2273
  @files.delete_files_from_server(deleted, commit_sha1)
2363
- progressbar.progress += deleted.size
2274
+ progressbar.finish
2364
2275
  res = @files.end_commit(commit_sha1, force: force, message: commit_msg)
2365
2276
  unless Cnvrg::CLI.is_response_success(res, false)
2366
2277
  raise StandardError.new("Cant end commit")
@@ -2614,6 +2525,9 @@ module Cnvrg
2614
2525
  if git or @project.is_git
2615
2526
  return download_in_git(commit)
2616
2527
  end
2528
+ if commit.present?
2529
+ return jump(commit)
2530
+ end
2617
2531
  ignore = options[:ignore] || ""
2618
2532
  if ignore.nil? or ignore.empty?
2619
2533
  ignore = ignore_list
@@ -2683,17 +2597,20 @@ module Cnvrg
2683
2597
 
2684
2598
 
2685
2599
  Cnvrg::Logger.log_info("Downloading updated files:#{updated_files.join(",")}")
2686
- @files.download_files(updated_files, commit, progress: progressbar) if updated_files.present?
2600
+ @files.download_files(updated_files, commit, progress: progressbar)
2687
2601
 
2688
2602
  Cnvrg::Logger.log_info("Downloading conflicted files:#{conflicted_files.join(",")}")
2689
- @files.download_files(conflicted_files, commit, postfix: ".conflict", progress: progressbar) if conflicted_files.present?
2603
+ @files.download_files(conflicted_files, commit, postfix: ".conflict", progress: progressbar)
2690
2604
 
2691
2605
  Cnvrg::Logger.log_info("Delete files: #{deleted_files.join(",")}")
2692
2606
  @files.delete_files_local(deleted_files, conflicted: conflicted_deleted, progress: progressbar)
2693
2607
 
2694
2608
  # update idx with latest commit
2695
- @project.update_idx_with_commit!(commit)
2609
+ # the latest true its because if we define --commit in the cmd it will go to "def jump(options['commit'])"
2610
+ # so if we are downloads something, we have to stay here.
2611
+ @project.update_idx_with_commit!(commit, latest: true)
2696
2612
  #TODO Sync, remove idx, sync again and pray
2613
+ progressbar.finish
2697
2614
  check = Helpers.checkmark()
2698
2615
  Cnvrg::Logger.log_info("Finished downloading successfuly")
2699
2616
  if options["verbose"]
@@ -2728,10 +2645,11 @@ module Cnvrg
2728
2645
  project_home = get_project_home
2729
2646
  @project = Project.new(project_home)
2730
2647
  current_commit = @project.last_local_commit
2731
- if current_commit.eql? commit_sha1
2648
+ if current_commit.start_with? commit_sha1 #commit_sha1 can be partial.
2732
2649
  log_message("Project is already updated", Thor::Shell::Color::GREEN)
2733
2650
  exit(0)
2734
2651
  end
2652
+ log_message("Jumping to commit #{commit_sha1}")
2735
2653
  @files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: project_home)
2736
2654
  resp = @project.jump_idx(destination: commit_sha1)
2737
2655
  if resp.blank?
@@ -2739,81 +2657,27 @@ module Cnvrg
2739
2657
  exit(0)
2740
2658
  end
2741
2659
  compare = resp['result']['compare']
2742
- resolver = resp['result']['resolver']
2743
2660
  latest = resp['result']['latest']
2744
2661
  commit = resp['result']['commit']
2745
- updated_files = compare['updated_on_server']
2662
+ updated_files = compare['updated_on_server'] + compare['added']
2746
2663
  conflicted_files = compare['conflicts']
2664
+ conflicted_deleted = compare['delete_conflicts'] || []
2747
2665
  deleted_files = compare['deleted']
2748
2666
  overall_changes = [updated_files, conflicted_files, deleted_files].flatten.size
2749
2667
 
2750
-
2751
- progressbar = ProgressBar.create(:title => "Download Progress",
2752
- :progress_mark => '=',
2753
- :format => "%b>>%i| %p%% %t",
2754
- :starting_at => 0,
2755
- :total => overall_changes,
2756
- :autofinish => true)
2757
-
2668
+ progressbar = @files.create_progressbar(overall_changes, "Download Progress")
2758
2669
  @files.download_files(updated_files, commit_sha1, progress: progressbar)
2759
2670
  @files.download_files(conflicted_files, commit_sha1, progress: progressbar, postfix: '.conflicted')
2760
- @files.delete_files_local(deleted_files, progress: progressbar)
2761
-
2762
-
2763
- @project.set_on_branch(latest)
2764
- @project.update_idx_with_commit!(commit)
2671
+ @files.delete_files_local(deleted_files, progress: progressbar, conflicted: conflicted_deleted)
2672
+ progressbar.finish
2673
+ @project.update_idx_with_commit!(commit, latest: latest)
2765
2674
  @project.generate_idx
2675
+
2766
2676
  log_message("Jumped successfuly!", Thor::Shell::Color::GREEN)
2767
2677
  rescue => e
2678
+ Logger::log_error(e)
2768
2679
  log_message("Cant jump to the specified commit", Thor::Shell::Color::RED)
2769
2680
  end
2770
- # successful_changes = []
2771
- #
2772
- # if !response["result"]["tree"].nil?
2773
-
2774
- # commit_sha1 = response["result"]["commit"]
2775
- # idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
2776
- # File.open(project_home + "/.cnvrg/idx.yml", "w+") {|f| f.write idx.to_yaml}
2777
- # if is_remote
2778
- # current_tree = Dir.glob("**/*", File::FNM_DOTMATCH).flatten.reject {|file| file.start_with? '.' or file.eql? "__init__.py" or file.eql? "uwsgi.ini" or file.ends_with? "/." or file.eql? "."}
2779
- # else
2780
- # current_tree = Dir.glob("**/*", File::FNM_DOTMATCH).flatten.reject {|file| file.start_with? '.cnvrg' or file.ends_with? "/." or file.eql? "."}
2781
- # end
2782
- #
2783
- # jump_result = Parallel.map(response["result"]["tree"], parallel_options) do |f|
2784
- #
2785
- # relative_path = f[0].gsub(/^#{@project.local_path}/, "")
2786
- # log_message("Downloading #{f[0]}", Thor::Shell::Color::BLUE, false)
2787
- # if f[0].end_with? "/"
2788
- # # dir
2789
- # @files.download_dir(f[0], relative_path, project_home)
2790
- #
2791
- # else
2792
- # # blob
2793
- # @files.download_file_s3(f[0], relative_path, project_home, commit_sha1 = commit_sha1)
2794
- # end
2795
- # end
2796
- #
2797
- #
2798
- # successful_changes = jump_result.select {|x| not x.nil?}
2799
- # end
2800
- # response["result"]["tree"].each do |f|
2801
- # if f[0].end_with? "/"
2802
- # current_tree.delete(f[0][0, f[0].size - 1])
2803
- # else
2804
- # current_tree.delete(f[0])
2805
- # end
2806
- # end
2807
- #
2808
- # FileUtils.rm_rf(current_tree)
2809
- # log_message("Done. Jumped to #{commit_sha1} completed successfully", Thor::Shell::Color::GREEN)
2810
- # rescue => e
2811
- # log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
2812
- # log_error(e)
2813
- # rescue SignalException
2814
- #
2815
- # exit(1)
2816
- # end
2817
2681
  end
2818
2682
 
2819
2683
  desc 'show', 'Show specific file from a specific commit'
@@ -2911,6 +2775,7 @@ module Cnvrg
2911
2775
  method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
2912
2776
  def sync(direct = true)
2913
2777
  verify_logged_in(true) if direct
2778
+ @project = Project.new(get_project_home)
2914
2779
  log_start(__method__, args, options)
2915
2780
  log_message('Checking for new updates from remote version', Thor::Shell::Color::BLUE, options["verbose"])
2916
2781
  log_message('Syncing project', Thor::Shell::Color::BLUE, !options["verbose"])
@@ -2919,7 +2784,7 @@ module Cnvrg
2919
2784
  in_exp = options["in_exp"] || (job_slug.present? and job_type.present?)
2920
2785
  in_exp = false if job_type.present? and job_type == "NotebookSession"
2921
2786
  run_download = true
2922
- if options[:force] or options[:files].present? or options[:output_dir].present? or in_exp
2787
+ if options[:force] or options[:files].present? or options[:output_dir].present? or in_exp or @project.is_branch
2923
2788
  run_download = false
2924
2789
  end
2925
2790
  if run_download
@@ -3477,10 +3342,6 @@ module Cnvrg
3477
3342
  else
3478
3343
  sync_result = invoke :sync, [false], :force => false
3479
3344
  end
3480
-
3481
-
3482
-
3483
-
3484
3345
  end
3485
3346
  #handle grid if it's git project
3486
3347
  if project.is_git and grid.present?
@@ -3516,6 +3377,7 @@ module Cnvrg
3516
3377
  if forced_commit and (commit_to_run.nil? or commit_to_run.empty?)
3517
3378
  commit_to_run = forced_commit
3518
3379
  end
3380
+
3519
3381
  commit_to_run = commit_to_run.presence || project.last_local_commit
3520
3382
 
3521
3383
  res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
@@ -5491,7 +5353,7 @@ module Cnvrg
5491
5353
  end
5492
5354
  end
5493
5355
 
5494
- def log_message(message, type, to_print = true)
5356
+ def log_message(message, type=Thor::Shell::Color::GREEN, to_print = true)
5495
5357
  if to_print
5496
5358
  say message, type
5497
5359
  end
@@ -5538,10 +5400,10 @@ module Cnvrg
5538
5400
  error = response['message'] || "Unknown error"
5539
5401
  # Cnvrg::CLI.log_end(1, error)
5540
5402
  if response["status"] == 500
5541
- say("<%= color('Server Error', RED) %>")
5403
+ log_message('Server Error', Thor::Shell::Color::RED)
5542
5404
  else
5543
5405
  $LOG.error message: error, type: "error"
5544
- say("<%= color('Error:#{error}', RED) %>")
5406
+ log_message("error", Thor::Shell::Color::RED)
5545
5407
  end
5546
5408
 
5547
5409
  if should_exit
@@ -5555,7 +5417,7 @@ module Cnvrg
5555
5417
  puts e.message
5556
5418
  puts e.backtrace
5557
5419
  rescue SignalException
5558
- say "Aborting"
5420
+ log_message "Aborting", Thor::Shell::Color::RED
5559
5421
  end
5560
5422
 
5561
5423
  end
@@ -5837,7 +5699,7 @@ module Cnvrg
5837
5699
  rescue
5838
5700
  return false
5839
5701
  rescue SignalException
5840
- say "\nAborting"
5702
+ say "\nAborting", Thor::Shell::Color::RED
5841
5703
  Exit(0)
5842
5704
  end
5843
5705
 
@@ -567,17 +567,15 @@ module Cnvrg
567
567
  end
568
568
 
569
569
 
570
- def download_file_s3(absolute_path, relative_path, project_home, commit_sha1=nil, conflict=false)
570
+ def download_file_s3(relative_path, commit_sha1=nil, postfix: '')
571
571
  begin
572
- res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path,
572
+ res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {relative_path: relative_path,
573
573
  commit_sha1: commit_sha1,new_version:true})
574
574
 
575
575
  Cnvrg::CLI.is_response_success(res, false)
576
576
  if res["result"]
577
577
  download_resp = res
578
578
  filename = download_resp["result"]["filename"]
579
-
580
- absolute_path += ".conflict" if conflict
581
579
  sts_path = download_resp["result"]["path_sts"]
582
580
  retries = 0
583
581
  success= false
@@ -632,8 +630,8 @@ module Cnvrg
632
630
  :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
633
631
  :http_open_timeout => 60, :retry_limit => 20)
634
632
  end
635
-
636
- File.open(project_home+"/"+absolute_path, 'w+') do |file|
633
+ absolute_path = "#{@project_home}/#{relative_path}#{postfix}"
634
+ File.open(absolute_path, 'w+') do |file|
637
635
  resp = client.get_object({bucket:bucket,
638
636
  key:file_key}, target: file)
639
637
  end
@@ -649,7 +647,21 @@ module Cnvrg
649
647
  end
650
648
  end
651
649
 
650
+ def create_progressbar(length = 10, title = 'Progress')
651
+ ProgressBar.create(:title => title,
652
+ :progress_mark => '=',
653
+ :format => "%b>>%i| %p%% %t",
654
+ :starting_at => 0,
655
+ :total => length,
656
+ :autofinish => true)
657
+ end
658
+
652
659
  def download_files(files, commit, postfix: '', progress: nil)
660
+ return if files.blank?
661
+ if Cnvrg::Helpers.server_version < 1
662
+ Cnvrg::Logger.log_info("Download files from older server.")
663
+ return self.download_files_old(files, commit, progress: progress, postfix: postfix)
664
+ end
653
665
  res = Cnvrg::API.request(@base_resource + "download_files", 'POST', {files: files, commit: commit})
654
666
  unless Cnvrg::CLI.is_response_success(res, false)
655
667
  raise Exception.new("Cant download files from the server.")
@@ -657,6 +669,14 @@ module Cnvrg
657
669
  self.download_multpile_files_s3(res['result'], @project_home, postfix: postfix, progress: progress)
658
670
  end
659
671
 
672
+
673
+ def download_files_old(files, commit, postfix: '', progress: nil)
674
+ files.each do |file|
675
+ self.download_file_s3(file, commit, postfix: postfix)
676
+ progress.progress += 1 if progress.present?
677
+ end
678
+ end
679
+
660
680
  def delete_files_local(deleted, conflicted: [], progress: nil)
661
681
  deleted -= conflicted
662
682
  deleted.each{|file| self.delete(file); progress.progress += 1 if progress.present?}
@@ -11,7 +11,7 @@ module Cnvrg
11
11
  begin
12
12
  @local_path = project_home
13
13
  @working_dir = project_home
14
- config = YAML.load_file(project_home+"/.cnvrg/config.yml")
14
+ config = YAML.load_file(project_home + "/.cnvrg/config.yml")
15
15
  @title = config[:project_name]
16
16
  @slug = config[:project_slug]
17
17
  @owner = config[:owner]
@@ -45,7 +45,7 @@ module Cnvrg
45
45
  end
46
46
  list = new_ignore.split(",")
47
47
  begin
48
- File.open(self.local_path+"/.cnvrgignore", "a+") do |f|
48
+ File.open(self.local_path + "/.cnvrgignore", "a+") do |f|
49
49
  f.puts("\n")
50
50
  list.each do |i|
51
51
  f.puts("#{i}\n")
@@ -59,16 +59,16 @@ module Cnvrg
59
59
 
60
60
  def get_ignore_list
61
61
  ignore_list = []
62
- if !File.exist? self.local_path+"/.cnvrgignore"
62
+ if !File.exist? self.local_path + "/.cnvrgignore"
63
63
  return ignore_list
64
64
  end
65
- File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
65
+ File.open(self.local_path + "/.cnvrgignore", "r").each_line do |line|
66
66
  line = line.strip
67
67
  if line.start_with? "#" or ignore_list.include? line or line.empty?
68
68
  next
69
69
  end
70
70
  if line.ends_with? "*"
71
- list_regex = Dir.glob("**/#{line}",File::FNM_DOTMATCH).flatten
71
+ list_regex = Dir.glob("**/#{line}", File::FNM_DOTMATCH).flatten
72
72
  list_regex.each do |l|
73
73
  ignore_list << l
74
74
  if File.directory?(l)
@@ -79,15 +79,15 @@ module Cnvrg
79
79
 
80
80
  end
81
81
  elsif line.ends_with? "/*"
82
- line = line.gsub("/*","")
82
+ line = line.gsub("/*", "")
83
83
  regex_list = Dir.glob("**/#{line}/**/*", File::FNM_DOTMATCH).flatten
84
84
  ignore_list << regex_list
85
85
  elsif line.include? "*"
86
- regex_list = Dir.glob("**/#{line}").flatten
87
- ignore_list << regex_list
86
+ regex_list = Dir.glob("**/#{line}").flatten
87
+ ignore_list << regex_list
88
88
  elsif line.end_with? "/" or File.directory?(line)
89
- ignore_list << line
90
- all_sub = Dir.glob("#{line}/**/*", File::FNM_DOTMATCH).flatten
89
+ ignore_list << line
90
+ all_sub = Dir.glob("#{line}/**/*", File::FNM_DOTMATCH).flatten
91
91
 
92
92
  ignore_list << all_sub.flatten
93
93
 
@@ -102,16 +102,16 @@ module Cnvrg
102
102
  def send_ignore_list()
103
103
  begin
104
104
  ignore_list = []
105
- File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
105
+ File.open(self.local_path + "/.cnvrgignore", "r").each_line do |line|
106
106
  line = line.strip
107
107
  if line.start_with? "#" or ignore_list.include? line or line.empty?
108
108
  next
109
109
  end
110
110
  if line.end_with? "/"
111
- ignore_list << line.gsub("/","")
112
- ignore_list << line+"."
111
+ ignore_list << line.gsub("/", "")
112
+ ignore_list << line + "."
113
113
  elsif line.include? "*"
114
- line = line.gsub("*",".*")
114
+ line = line.gsub("*", ".*")
115
115
  ignore_list << line
116
116
  else
117
117
  ignore_list << line
@@ -126,7 +126,7 @@ module Cnvrg
126
126
 
127
127
  # Create project
128
128
 
129
- def self.create(project_name, clean, with_docker=false)
129
+ def self.create(project_name, clean, with_docker = false)
130
130
  if clean
131
131
  list_dirs = [project_name, project_name + "/.cnvrg"]
132
132
  else
@@ -149,7 +149,7 @@ module Cnvrg
149
149
  ]
150
150
  cnvrgreadme = Helpers.readme_content
151
151
  cnvrgignore = Helpers.cnvrgignore_content
152
- cnvrghyper = Helpers.hyper_content
152
+ cnvrghyper = Helpers.hyper_content
153
153
 
154
154
  begin
155
155
 
@@ -166,10 +166,10 @@ module Cnvrg
166
166
  FileUtils.mkdir_p list_dirs
167
167
  FileUtils.touch list_files
168
168
 
169
- File.open(project_name + "/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
170
- File.open(project_name + "/.cnvrgignore", "w+") { |f| f.write cnvrgignore }
171
- File.open(project_name + "/README.md", "w+") { |f| f.write cnvrgreadme }
172
- File.open(project_name + "/src/hyper.yaml", "w+") { |f| f.write cnvrghyper }
169
+ File.open(project_name + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
170
+ File.open(project_name + "/.cnvrgignore", "w+") {|f| f.write cnvrgignore}
171
+ File.open(project_name + "/README.md", "w+") {|f| f.write cnvrgreadme}
172
+ File.open(project_name + "/src/hyper.yaml", "w+") {|f| f.write cnvrghyper}
173
173
 
174
174
  rescue
175
175
  return false
@@ -177,75 +177,76 @@ module Cnvrg
177
177
  return true
178
178
  end
179
179
 
180
- def self.link(owner, project_name, docker=false, git = false)
181
- ignore_exits = File.exist? ".cnvrgignore"
182
- list_dirs = [".cnvrg"
183
- ]
184
- list_files = [
185
- ".cnvrg/config.yml"
186
- ]
187
- if !ignore_exits
188
- list_files <<
189
- ".cnvrgignore"
190
- end
180
+ def self.link(owner, project_name, docker = false, git = false)
181
+ ignore_exits = File.exist? ".cnvrgignore"
182
+ list_dirs = [".cnvrg"
183
+ ]
184
+ list_files = [
185
+ ".cnvrg/config.yml"
186
+ ]
187
+ if !ignore_exits
188
+ list_files <<
189
+ ".cnvrgignore"
190
+ end
191
191
 
192
- cnvrgreadme = Helpers.readme_content
193
- cnvrgignore = Helpers.cnvrgignore_content
194
- begin
195
- response = Cnvrg::API.request("cli/create_project", 'POST', {title: project_name, owner: owner, is_docker: docker})
196
- Cnvrg::CLI.is_response_success(response)
197
- response = JSON.parse response["result"]
198
- project_slug = response["slug"]
199
-
200
- config = {project_name: project_name,
201
- project_slug: project_slug,
202
- owner: owner,
203
- git: git}
204
- FileUtils.mkdir_p list_dirs
205
- FileUtils.touch list_files
206
- File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
207
- File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless ignore_exits
208
- if !File.exist? "README" and !File.exist? "README.md"
209
- FileUtils.touch [ "README.md" ]
210
- File.open("README.md", "w+") { |f| f.write cnvrgreadme }
211
- end
192
+ cnvrgreadme = Helpers.readme_content
193
+ cnvrgignore = Helpers.cnvrgignore_content
194
+ begin
195
+ response = Cnvrg::API.request("cli/create_project", 'POST', {title: project_name, owner: owner, is_docker: docker})
196
+ Cnvrg::CLI.is_response_success(response)
197
+ response = JSON.parse response["result"]
198
+ project_slug = response["slug"]
212
199
 
213
- rescue => e
214
- puts e
215
- return false
200
+ config = {project_name: project_name,
201
+ project_slug: project_slug,
202
+ owner: owner,
203
+ git: git}
204
+ FileUtils.mkdir_p list_dirs
205
+ FileUtils.touch list_files
206
+ File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
207
+ File.open(".cnvrgignore", "w+") {|f| f.write cnvrgignore} unless ignore_exits
208
+ if !File.exist? "README" and !File.exist? "README.md"
209
+ FileUtils.touch ["README.md"]
210
+ File.open("README.md", "w+") {|f| f.write cnvrgreadme}
216
211
  end
217
- return true
212
+
213
+ rescue => e
214
+ puts e
215
+ return false
218
216
  end
217
+ return true
218
+ end
219
219
 
220
- def self.clone_dir(project_slug, project_owner, project_name,is_git=false)
221
- list_dirs = [project_name,
222
- project_name + "/.cnvrg"
223
- ]
220
+ def self.clone_dir(project_slug, project_owner, project_name, is_git = false)
221
+ list_dirs = [project_name,
222
+ project_name + "/.cnvrg"
223
+ ]
224
224
 
225
225
 
226
- list_files = [
227
- project_name + "/.cnvrg/config.yml",
228
- project_name+"/.cnvrgignore",
229
- ]
230
- begin
231
- config = {project_name: project_name,
232
- project_slug: project_slug,
233
- owner: project_owner,
234
- git: is_git}
235
- FileUtils.mkdir_p list_dirs
236
- FileUtils.touch list_files
237
- cnvrgignore = Helpers.cnvrgignore_content
226
+ list_files = [
227
+ project_name + "/.cnvrg/config.yml",
228
+ project_name + "/.cnvrgignore",
229
+ ]
230
+ begin
231
+ config = {project_name: project_name,
232
+ project_slug: project_slug,
233
+ owner: project_owner,
234
+ git: is_git}
235
+ FileUtils.mkdir_p list_dirs
236
+ FileUtils.touch list_files
237
+ cnvrgignore = Helpers.cnvrgignore_content
238
238
 
239
239
 
240
- File.open(project_name + "/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
241
- File.open(project_name+"/.cnvrgignore", "w+") { |f| f.write cnvrgignore }
240
+ File.open(project_name + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
241
+ File.open(project_name + "/.cnvrgignore", "w+") {|f| f.write cnvrgignore}
242
242
 
243
- rescue
244
- return false
245
- end
246
- return true
243
+ rescue
244
+ return false
247
245
  end
248
- def self.verify_cnvrgignore_exist(project_name,remote)
246
+ return true
247
+ end
248
+
249
+ def self.verify_cnvrgignore_exist(project_name, remote)
249
250
  path = ".cnvrgignore"
250
251
  if !File.exist? path
251
252
  path = "#{project_name}/.cnvrgignore"
@@ -255,10 +256,10 @@ module Cnvrg
255
256
  begin
256
257
  list_files = [
257
258
  path
258
- ]
259
- FileUtils.touch list_files
260
- cnvrgignore = Helpers.cnvrgignore_content
261
- File.open(path, "w+") { |f| f.write cnvrgignore }
259
+ ]
260
+ FileUtils.touch list_files
261
+ cnvrgignore = Helpers.cnvrgignore_content
262
+ File.open(path, "w+") {|f| f.write cnvrgignore}
262
263
  rescue => e
263
264
  return false
264
265
  end
@@ -266,64 +267,66 @@ module Cnvrg
266
267
  end
267
268
  end
268
269
 
269
- def self.clone_dir_remote(project_slug, project_owner, project_name,is_git=false)
270
- list_dirs = [
271
- ".cnvrg"
272
- ]
273
-
274
-
270
+ def self.clone_dir_remote(project_slug, project_owner, project_name, is_git = false)
271
+ list_dirs = [
272
+ ".cnvrg"
273
+ ]
275
274
 
276
- list_files = [
277
- ".cnvrg/config.yml",
278
275
 
279
- ]
276
+ list_files = [
277
+ ".cnvrg/config.yml",
280
278
 
279
+ ]
281
280
 
282
- begin
283
- config = {project_name: project_name,
284
- project_slug: project_slug,
285
- owner: project_owner,
286
- git: is_git
287
- }
288
- FileUtils.mkdir_p list_dirs
289
- FileUtils.touch list_files
290
281
 
282
+ begin
283
+ config = {project_name: project_name,
284
+ project_slug: project_slug,
285
+ owner: project_owner,
286
+ git: is_git
287
+ }
288
+ FileUtils.mkdir_p list_dirs
289
+ FileUtils.touch list_files
291
290
 
292
- File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
293
- if !File.exist? ".cnvrgignore"
294
- FileUtils.touch ".cnvrgignore"
295
- list_files << ".cnvrgignore"
296
- cnvrgignore = Helpers.cnvrgignore_content
297
- File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore }
298
291
 
292
+ File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
293
+ if !File.exist? ".cnvrgignore"
294
+ FileUtils.touch ".cnvrgignore"
295
+ list_files << ".cnvrgignore"
296
+ cnvrgignore = Helpers.cnvrgignore_content
297
+ File.open(".cnvrgignore", "w+") {|f| f.write cnvrgignore}
299
298
 
300
- end
301
299
 
302
- rescue
303
- return false
304
300
  end
305
- return true
301
+
302
+ rescue
303
+ return false
306
304
  end
305
+ return true
306
+ end
307
+
307
308
  def update_is_new_branch(new_branch)
308
- config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
309
+ config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
309
310
  config[:new_branch] = new_branch
310
- File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
311
+ File.open(@working_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
311
312
  end
313
+
312
314
  def get_new_branch
313
315
  begin
314
- config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
315
- return config[:new_branch]
316
- rescue =>e
316
+ config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
317
+ return config[:new_branch]
318
+ rescue => e
317
319
  return false
318
320
 
319
321
  end
320
322
  end
321
323
 
322
324
  def remove_new_branch
323
- config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
325
+ config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
324
326
  new_config = config.except(:new_branch)
325
- File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write new_config.to_yaml }
327
+ File.open(@working_dir + "/.cnvrg/config.yml", "w+") {|f| f.write new_config.to_yaml}
326
328
  end
329
+
327
330
  def generate_output_dir(output_dir)
328
331
  Cnvrg::Logger.log_info("Generating output dir for #{output_dir}")
329
332
  upload_list = []
@@ -332,10 +335,10 @@ module Cnvrg
332
335
  next if e.end_with? "/."
333
336
  if File.directory? e
334
337
 
335
- upload_list << e+"/"
338
+ upload_list << e + "/"
336
339
  else
337
340
  upload_list << e
338
- end
341
+ end
339
342
  end
340
343
  if Dir.exists? output_dir
341
344
  upload_list << output_dir + "/"
@@ -364,130 +367,132 @@ module Cnvrg
364
367
 
365
368
  end
366
369
 
367
- def generate_idx(deploy:false)
368
- if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
369
- old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
370
- else
371
- old_idx = {:tree => {}, :commit => nil}
372
- end
370
+ def generate_idx(deploy: false)
371
+ if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
372
+ old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
373
+ else
374
+ old_idx = {:tree => {}, :commit => nil}
375
+ end
373
376
 
374
- tree_idx = Hash.new(0)
375
- list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~ /^#{self.local_path}\/\.git\/*/) or (x =~/^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~/^#{self.local_path}\/\.cnvrgignore/) }
376
- list_ignore = self.get_ignore_list()
377
- if deploy
378
- list_ignore << ["main.py","main.pyc", "__init__.py", "uwsgi.ini"]
379
- list_ignore.flatten!
380
- end
377
+ tree_idx = Hash.new(0)
378
+ list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject {|x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~ /^#{self.local_path}\/\.git\/*/) or (x =~ /^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~ /^#{self.local_path}\/\.cnvrgignore/)}
379
+ list_ignore = self.get_ignore_list()
380
+ if deploy
381
+ list_ignore << ["main.py", "main.pyc", "__init__.py", "uwsgi.ini"]
382
+ list_ignore.flatten!
383
+ end
381
384
 
382
- Parallel.map(list, in_threads: IDXParallelThreads) do |e|
383
- label = e.gsub(self.local_path + "/", "")
384
- ignore_label = label.gsub("/","//")
385
- if list_ignore.include? ignore_label
386
- next
387
- end
388
- if File.directory? e
385
+ Parallel.map(list, in_threads: IDXParallelThreads) do |e|
386
+ label = e.gsub(self.local_path + "/", "")
387
+ ignore_label = label.gsub("/", "//")
388
+ if list_ignore.include? ignore_label
389
+ next
390
+ end
391
+ if File.directory? e
389
392
 
390
- tree_idx[label+"/"] = nil
393
+ tree_idx[label + "/"] = nil
394
+ else
395
+ sha1 = OpenSSL::Digest::SHA1.file(e).hexdigest
396
+ if old_idx.nil? or old_idx.to_h[:tree].nil?
397
+ tree_idx[label] = {sha1: sha1, commit_time: nil}
398
+ elsif old_idx[:tree][label].nil? or old_idx[:tree][label][:sha1] != sha1
399
+ tree_idx[label] = {sha1: sha1, commit_time: nil}
391
400
  else
392
- sha1 = OpenSSL::Digest::SHA1.file(e).hexdigest
393
- if old_idx.nil? or old_idx.to_h[:tree].nil?
394
- tree_idx[label] = {sha1: sha1, commit_time: nil}
395
- elsif old_idx[:tree][label].nil? or old_idx[:tree][label][:sha1] != sha1
396
- tree_idx[label] = {sha1: sha1, commit_time: nil}
397
- else
398
- tree_idx[label] = old_idx[:tree][label]
399
- end
401
+ tree_idx[label] = old_idx[:tree][label]
400
402
  end
401
403
  end
404
+ end
402
405
 
403
- old_idx[:tree] = tree_idx
406
+ old_idx[:tree] = tree_idx
404
407
 
405
- File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write old_idx.to_yaml }
406
- return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
407
- end
408
- def get_idx
409
- unless File.exists? "#{self.local_path}/.cnvrg/idx.yml"
410
- empty_idx = {:commit => nil, :tree => {}}
411
- File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write empty_idx.to_yaml }
412
- return empty_idx
413
- end
414
- YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
408
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write old_idx.to_yaml}
409
+ return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
410
+ end
411
+
412
+ def get_idx
413
+ unless File.exists? "#{self.local_path}/.cnvrg/idx.yml"
414
+ empty_idx = {:commit => nil, :tree => {}}
415
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write empty_idx.to_yaml}
416
+ return empty_idx
415
417
  end
418
+ YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
419
+ end
416
420
 
417
421
  def set_idx(idx)
418
422
  FileUtils.mkdir_p("#{self.local_path}/.cnvrg")
419
- File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
423
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx.to_yaml}
424
+ end
425
+
426
+ def clone(remote = 0, commit)
427
+ response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/clone", 'POST', {project_slug: self.slug, remote: remote, commit: commit})
428
+ return response
420
429
  end
421
430
 
422
- def clone(remote=0, commit)
423
- response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/clone", 'POST', {project_slug: self.slug, remote: remote, commit: commit})
424
- return response
425
- end
426
431
  def git_download_commit(commit)
427
432
  response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/git_download_commit", 'POST', {commit_sha1: commit})
428
- CLI.is_response_success(response,true)
433
+ CLI.is_response_success(response, true)
429
434
  return response
430
435
  end
431
436
 
432
437
  def compare_idx(new_branch, force:false, deploy: false, in_exp:false, specific_files: [], download: false)
433
- is_download = download
434
- if is_download
435
- local_idx = self.get_idx
436
- else
437
- #upload
438
- local_idx = self.generate_idx(deploy: deploy)
439
- end
440
- commit = local_idx[:commit]
441
- tree = local_idx[:tree]
442
- ignore_list = self.send_ignore_list()
443
- if force or specific_files.present?
444
- added = []
445
- if specific_files.present?
446
- added = specific_files
447
- elsif tree.present?
448
- added << local_idx[:tree].keys
449
- added.flatten!
450
- end
451
- response ={"result"=> {"commit"=>nil,"tree"=> {"added"=> added,
452
- "updated_on_server"=> [],
453
- "updated_on_local"=> [],
454
- "update_local" => [],
455
- "deleted"=> [],
456
- "conflicts"=> []} } }
457
- return response
438
+ is_download = download
439
+ if is_download
440
+ local_idx = self.get_idx
441
+ else
442
+ #upload
443
+ local_idx = self.generate_idx(deploy: deploy)
444
+ end
445
+ commit = local_idx[:commit]
446
+ tree = local_idx[:tree]
447
+ ignore_list = self.send_ignore_list()
448
+ if force or specific_files.present?
449
+ added = []
450
+ if specific_files.present?
451
+ added = specific_files
452
+ elsif tree.present?
453
+ added << local_idx[:tree].keys
454
+ added.flatten!
458
455
  end
459
- #we dont want to send it on download - we only compare between commits sha1 in download.
460
- if is_download
461
- #the new server doesnt need the tree, but the old probably needs :X
462
- local_idx[:tree] = {} if Cnvrg::Helpers.server_version > 0
456
+ response = {"result" => {"commit" => nil, "tree" => {"added" => added,
457
+ "updated_on_server" => [],
458
+ "updated_on_local" => [],
459
+ "update_local" => [],
460
+ "deleted" => [],
461
+ "conflicts" => []}}}
462
+ return response
463
+ end
464
+ #we dont want to send it on download - we only compare between commits sha1 in download.
465
+ if is_download
466
+ #the new server doesnt need the tree, but the old probably needs :X
467
+ local_idx[:tree] = {} if Cnvrg::Helpers.server_version > 0
468
+ end
469
+ response = Cnvrg::API.request(@base_resource + "status", 'POST', {idx: local_idx, new_branch: new_branch,
470
+ current_commit: commit, ignore: ignore_list, force: force, in_exp: in_exp, download: download})
471
+
472
+ CLI.is_response_success(response, true)
473
+ if is_download
474
+ if Cnvrg::Helpers.server_version > 0
475
+ #trying to optimize the download using resolver
476
+ resolve = response['result']['tree']['resolver'] || tree #tree of file -> sha1 from current commit to check conflicts
477
+ destination_files = response['result']['tree']['destination'] || {} #tree of file -> sha1 from last commit to check files that already downloaded
478
+ else
479
+ resolve = tree
480
+ destination_files = {}
463
481
  end
464
- response = Cnvrg::API.request(@base_resource + "status", 'POST', {idx: local_idx, new_branch: new_branch,
465
- current_commit: commit,ignore:ignore_list, force:force,in_exp:in_exp, download: download})
466
-
467
- CLI.is_response_success(response,true)
468
- if is_download
469
- if Cnvrg::Helpers.server_version > 0
470
- #trying to optimize the download using resolver
471
- resolve = response['result']['tree']['resolver'] || tree #tree of file -> sha1 from current commit to check conflicts
472
- destination_files = response['result']['tree']['destination'] || {} #tree of file -> sha1 from last commit to check files that already downloaded
473
- else
474
- resolve = tree
475
- destination_files = {}
476
- end
477
- @files = self.get_files
478
- local_tree = @files.calculate_sha1(resolve.keys)
479
- changed_files = resolve.keys.select{|file| local_tree[file] != resolve[file]}
482
+ @files = self.get_files
483
+ local_tree = @files.calculate_sha1(resolve.keys)
484
+ changed_files = resolve.keys.select {|file| local_tree[file] != resolve[file]}
480
485
 
481
- # means that the user changed the file locally
482
- response['result']['tree']['update_local'] = changed_files
486
+ # means that the user changed the file locally
487
+ response['result']['tree']['update_local'] = changed_files
483
488
 
484
- # means that we already downloaded this file and we dont need it anymore
485
- downloaded_files = destination_files.keys.select{|file| local_tree[file] == destination_files[file]}
486
- response['result']['tree']['added'] -= downloaded_files
487
- response['result']['tree']['updated_on_server'] -= downloaded_files
488
- end
489
- return response
489
+ # means that we already downloaded this file and we dont need it anymore
490
+ downloaded_files = destination_files.keys.select {|file| local_tree[file] == destination_files[file]}
491
+ response['result']['tree']['added'] -= downloaded_files
492
+ response['result']['tree']['updated_on_server'] -= downloaded_files
490
493
  end
494
+ return response
495
+ end
491
496
 
492
497
  def get_files
493
498
  Cnvrg::Files.new(self.owner, self.slug, project_home: @local_path)
@@ -502,52 +507,52 @@ module Cnvrg
502
507
  'POST',
503
508
  {tree: tree, ignore: ignore_list,
504
509
  dest_commit: destination, current_commit: current_commit})
505
- CLI.is_response_success(response,false)
510
+ CLI.is_response_success(response, false)
506
511
  response
507
512
  end
508
513
 
509
514
  def set_on_branch(is_latest)
510
- config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
515
+ config = YAML.load_file(@working_dir + "/.cnvrg/config.yml")
511
516
  @is_branch = !is_latest
512
517
  config[:is_branch] = @is_branch
513
518
 
514
- File.open(@working_dir+"/.cnvrg/config.yml", 'w') { |f| f.write config.to_yaml }
519
+ File.open(@working_dir + "/.cnvrg/config.yml", 'w') {|f| f.write config.to_yaml}
515
520
 
516
521
  end
517
522
 
518
- def compare_commit(commit)
519
- if commit.nil? or commit.empty?
520
- commit = last_local_commit
521
- end
522
- response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
523
- CLI.is_response_success(response,false)
524
- update_is_new_branch(response["result"]["new_branch"])
525
- return response["result"]["new_branch"]
523
+ def compare_commit(commit)
524
+ if commit.nil? or commit.empty?
525
+ commit = last_local_commit
526
526
  end
527
+ response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
528
+ CLI.is_response_success(response, false)
529
+ update_is_new_branch(response["result"]["new_branch"])
530
+ return response["result"]["new_branch"]
531
+ end
527
532
 
528
- def update_idx_with_files_commits!(files, commit_time)
529
-
530
- idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
531
- files.each do |path|
532
- idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
533
- end
534
- File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
533
+ def update_idx_with_files_commits!(files, commit_time)
535
534
 
536
- return true
535
+ idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
536
+ files.each do |path|
537
+ idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
537
538
  end
539
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
540
+
541
+ return true
542
+ end
538
543
 
539
- def deploy(file_to_run, function, input_params, commit_to_run, instance_type, image_slug, scheduling_query, local_timestamp,workers, file_input, title)
544
+ def deploy(file_to_run, function, input_params, commit_to_run, instance_type, image_slug, scheduling_query, local_timestamp, workers, file_input, title)
545
+ response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/deploy", 'POST', {file_to_run: file_to_run, function: function,
546
+ image_slug: image_slug, input_params: input_params,
547
+ commit_sha1: commit_to_run,
548
+ instance_type: instance_type,
549
+ scheduling_query: scheduling_query,
550
+ local_timestamp: local_timestamp,
551
+ workers: workers, file_input: file_input,
552
+ title: title})
553
+ return response
554
+ end
540
555
 
541
- response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/deploy", 'POST', {file_to_run: file_to_run, function: function,
542
- image_slug: image_slug, input_params: input_params,
543
- commit_sha1: commit_to_run,
544
- instance_type: instance_type,
545
- scheduling_query: scheduling_query,
546
- local_timestamp: local_timestamp,
547
- workers:workers,file_input:file_input,
548
- title: title})
549
- return response
550
- end
551
556
  def list_commits
552
557
  response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commits/list", 'GET')
553
558
  CLI.is_response_success(response)
@@ -567,17 +572,19 @@ module Cnvrg
567
572
  CLI.is_response_success(response)
568
573
  return response
569
574
  end
570
- def update_idx_with_commit!(commit)
571
- idx_hash = self.get_idx
572
- idx_hash[:commit] = commit
573
575
 
574
- File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
575
- return true
576
- end
576
+ def update_idx_with_commit!(commit, latest: nil)
577
+ idx_hash = self.get_idx
578
+ idx_hash[:commit] = commit
579
+ self.set_on_branch(latest) unless latest.nil?
577
580
 
578
- def revert(working_dir)
579
- FileUtils.rm_rf working_dir
580
- end
581
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
582
+ return true
583
+ end
584
+
585
+ def revert(working_dir)
586
+ FileUtils.rm_rf working_dir
587
+ end
581
588
 
582
589
  def init_machines
583
590
  Cnvrg::Logger.log_info("Init machines.")
@@ -599,5 +606,6 @@ module Cnvrg
599
606
  machines.include? machine
600
607
  end
601
608
 
602
- end
609
+
603
610
  end
611
+ end
@@ -1,4 +1,4 @@
1
1
  module Cnvrg
2
- VERSION = '0.8.5'
2
+ VERSION = '0.8.6'
3
3
  end
4
4
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cnvrg
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.5
4
+ version: 0.8.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yochay Ettun