cnvrg 0.0.1411 → 0.0.1412

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 093f03d38281b47f7a3f251b5875aa89ac27a39c
4
- data.tar.gz: 45a9001d2a87bdae1f7e097975a56b2c3b57be9d
3
+ metadata.gz: af467f0b591d7f5b4c48f1ea659ce6a8af10fa4a
4
+ data.tar.gz: a24c8338c54b12502ad9139f0fd12bd8865c5dc1
5
5
  SHA512:
6
- metadata.gz: 50991f6a61b5686e956dcd12baa0ca2bd1bba5ed826c73181144951949a6cda3a2e2dbceb75111cc26702c1b99ade85f74e640dc53dc3df46e28a0cbfb893ed9
7
- data.tar.gz: 153369319301d57aecbd07022ea224f7b2cadb6a56bb1365f9a42a5fa86bf0b8cc3593de05eb6bea0ade20dd0fbd50820a7313439ab7678063ab5a9437385897
6
+ metadata.gz: 560278384f51b69ec1a01c629a9089e48d93b5c6630ae5e406f4de697e593cd6b7368c1634f7a66a8d0f0c6446823f3682121ac14fc936738b2d3b87d8dbbd22
7
+ data.tar.gz: 76a8f5b59fe26c179496105ee28efd6103ccf32abfb088759424c4bf52a15dc7b8a7934cf8f72cf8cd7e50323e9455dda0ac02b7283dcada0478a0eb71b2ac01
data/lib/cnvrg/cli.rb CHANGED
@@ -20,6 +20,7 @@ require 'cnvrg/experiment'
20
20
  require 'cnvrg/Images'
21
21
  require 'cnvrg/dataset'
22
22
  require 'cnvrg/datafiles'
23
+ require 'cnvrg/data'
23
24
  require 'etc'
24
25
  require 'logstash-logger'
25
26
  require 'cnvrg/job'
@@ -32,12 +33,87 @@ require 'zip'
32
33
  require 'active_support/all'
33
34
  require 'thor'
34
35
  require 'pathname'
35
- require 'cnvrg/data'
36
- # DEV VERSION
37
36
 
38
- # include Cnvrg::Tar
39
37
 
40
- # include Tar
38
+ class Thor
39
+ module Base
40
+ def initialize(args = [], local_options = {}, config = {})
41
+ parse_options = Thor.class_options
42
+
43
+ # The start method splits inbound arguments at the first argument
44
+ # that looks like an option (starts with - or --). It then calls
45
+ # new, passing in the two halves of the arguments Array as the
46
+ # first two parameters.
47
+
48
+ command_options = config.delete(:command_options) # hook for start
49
+ parse_options = parse_options.merge(command_options) if command_options
50
+ if local_options.is_a?(Array)
51
+ array_options = local_options
52
+ hash_options = {}
53
+ else
54
+ # Handle the case where the class was explicitly instantiated
55
+ # with pre-parsed options.
56
+ array_options = []
57
+ hash_options = local_options
58
+ end
59
+
60
+ # Let Thor::Options parse the options first, so it can remove
61
+ # declared options from the array. This will leave us with
62
+ # a list of arguments that weren't declared.
63
+ stop_on_unknown = Thor.stop_on_unknown_option? config[:current_command]
64
+ opts = Thor::Options.new(parse_options, hash_options, stop_on_unknown)
65
+ real_options = []
66
+ if args.empty?
67
+ real_args = [].replace(array_options)
68
+
69
+ array_options.each_with_index do |p,i|
70
+ if !p.starts_with? "--" and !p.starts_with? "-"
71
+ break
72
+ else
73
+ real_options << p
74
+ if !p.include? "="
75
+ if i+1< array_options.size
76
+ real_options << array_options[i+1]
77
+ real_args.delete(array_options[i+1])
78
+ end
79
+ end
80
+ real_args.delete(p)
81
+
82
+ end
83
+ end
84
+ args = real_args
85
+ else
86
+ if local_options.is_a? Array
87
+ args = [args + local_options].flatten
88
+ end
89
+
90
+ end
91
+
92
+ self.options = opts.parse(real_options)
93
+ self.options = config[:class_options].merge(options) if config[:class_options]
94
+
95
+ # If unknown options are disallowed, make sure that none of the
96
+ # remaining arguments looks like an option.
97
+ opts.check_unknown! if Thor.check_unknown_options?(config)
98
+
99
+ # Add the remaining arguments from the options parser to the
100
+ # arguments passed in to initialize. Then remove any positional
101
+ # arguments declared using #argument (this is primarily used
102
+ # by Thor::Group). Tis will leave us with the remaining
103
+ # positional arguments.
104
+ to_parse = args
105
+ to_parse += opts.remaining unless self.class.strict_args_position?(config)
106
+ thor_args = Thor::Arguments.new(self.class.arguments)
107
+ thor_args.parse(to_parse).each { |k, v| __send__("#{k}=", v) }
108
+ @args = thor_args.remaining
109
+ end
110
+
111
+ end
112
+ end
113
+
114
+
115
+
116
+
41
117
  module Cnvrg
42
118
 
43
119
  class CLI < Thor
@@ -337,7 +413,6 @@ module Cnvrg
337
413
 
338
414
  def me()
339
415
  begin
340
-
341
416
  verify_logged_in(false)
342
417
  log_start(__method__, args, options)
343
418
  auth = Cnvrg::Auth.new
@@ -348,6 +423,7 @@ module Cnvrg
348
423
  end
349
424
 
350
425
  log_end(0)
426
+
351
427
  rescue SignalException
352
428
  log_end(-1)
353
429
 
@@ -593,7 +669,7 @@ module Cnvrg
593
669
  url = @dataset.url
594
670
  check = Helpers.checkmark
595
671
  say "#{check} Link finished successfully", Thor::Shell::Color::GREEN
596
- # say "#{dataset_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN
672
+ say "#{dataset_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN
597
673
  log_end(0)
598
674
 
599
675
  else
@@ -645,7 +721,7 @@ module Cnvrg
645
721
  say "Downloading data", Thor::Shell::Color::BLUE
646
722
 
647
723
 
648
- download_data(false, false, path = working_dir)
724
+ download_data(false, false, path = working_dir,in_dir=false)
649
725
 
650
726
 
651
727
  check = Helpers.checkmark
@@ -735,9 +811,9 @@ module Cnvrg
735
811
  method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
736
812
  method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
737
813
 
738
- def download_data(verbose, sync, path=Dir.pwd)
814
+ def download_data(verbose, sync, path=Dir.pwd,in_dir=true)
739
815
  begin
740
- verify_logged_in(true)
816
+ verify_logged_in(in_dir)
741
817
  log_start(__method__, args, options)
742
818
  if path.nil? or path.empty?
743
819
  path = Dir.pwd
@@ -1082,13 +1158,15 @@ module Cnvrg
1082
1158
  home_dir = File.expand_path('~')
1083
1159
  tar_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.tar.gz"
1084
1160
  tar_files_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.txt"
1085
- tar_files = (result["added"] + result["updated_on_local"] +["\n"]).join("\n")
1161
+ tar_files = (result["added"] + result["updated_on_local"]).join("\n")
1086
1162
  File.open(tar_files_path, 'w') { |f| f.write tar_files }
1087
1163
  is_tar = create_tar(dataset_dir, tar_path, tar_files_path)
1088
1164
  if !is_tar
1089
1165
  say "ERROR: Couldn't compress data", Thor::Shell::Color::RED
1090
1166
  FileUtils.rm_rf([tar_path]) if File.exist? tar_path
1091
1167
  FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
1168
+ @files.rollback_commit(commit_sha1)
1169
+ say "Rolling Back all changes.", Thor::Shell::Color::RED
1092
1170
  exit(1)
1093
1171
  end
1094
1172
  say "Uploading data", Thor::Shell::Color::BLUE
@@ -1194,7 +1272,7 @@ module Cnvrg
1194
1272
 
1195
1273
  end
1196
1274
 
1197
- desc 'list data', 'List all dataset you currently have'
1275
+ desc 'data list', 'List all dataset you currently have'
1198
1276
 
1199
1277
  def list_dataset
1200
1278
  verify_logged_in(false)
@@ -1212,7 +1290,7 @@ module Cnvrg
1212
1290
 
1213
1291
  end
1214
1292
 
1215
- desc 'list data commits', 'List all commits for a specific dataset'
1293
+ desc 'data commits', 'List all commits for a specific dataset'
1216
1294
 
1217
1295
  def list_dataset_commits()
1218
1296
  verify_logged_in(true)
@@ -1952,7 +2030,7 @@ module Cnvrg
1952
2030
  method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
1953
2031
  method_option :sync_before, :type => :boolean, :aliases => ["-sb"], :default => true
1954
2032
  method_option :sync_after, :type => :boolean, :aliases => ["-sa"], :default => true
1955
- method_option :title, :type => :string, :aliases => ["-t"], :default => ""
2033
+ method_option :title, :type => :string, :aliases => ["-t", "--title"], :default => ""
1956
2034
  method_option :log, :type => :boolean, :aliases => ["--log"], :default => false
1957
2035
  method_option :email_notification, :type => :boolean, :aliases => ["-en"], :default => false
1958
2036
  method_option :upload_output, :type => :string, :aliases => ["-uo"], :default => ""
@@ -1979,21 +2057,45 @@ module Cnvrg
1979
2057
  data = options["data"]
1980
2058
  data_commit = options["data_commit"]
1981
2059
  ignore = options["ignore"]
1982
-
2060
+ options_hash = Hash[options]
2061
+ real_options = []
2062
+ options_hash.each do |o|
2063
+ real_options << o if (!o[1].eql? "" and !["small","medium","large","gpu","gpuxl","gpuxxl"].include? o[0])
2064
+ end
1983
2065
  if local
1984
- invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
1985
- :log => log, :email_notification => email_notification, :upload_output => upload_output,
1986
- :commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore
1987
- return
2066
+ exec_options = real_options.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
2067
+ cmd_to_exec = "#{exec_options} #{cmd.join(" ")}"
2068
+ return exec(cmd_to_exec)
1988
2069
  else
2070
+ real_options.delete(["local",false])
1989
2071
  instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
1990
- "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
2072
+ "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
1991
2073
  instance_type = get_instance_type(instances)
1992
- invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
1993
- :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
1994
- :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore
1995
- return
2074
+ if !instance_type.nil? and !instance_type.empty?
2075
+ real_options << ["machine_type",instance_type]
2076
+ end
2077
+ exec_options = real_options.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
2078
+ cmd_to_exec = "#{exec_options} #{cmd.join(" ")}"
2079
+ return exec_remote(cmd_to_exec)
2080
+ # invoke :exec_remote, [cmd_to_exec], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
2081
+ # :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
2082
+ # :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore
2083
+ # return
1996
2084
  end
2085
+
2086
+ # if local
2087
+ # invoke :exec, [cmd_to_exec.split(" ")], :sync_before => sync_before, :sync_after => sync_after, :title => title,
2088
+ # :log => log, :email_notification => email_notification, :upload_output => upload_output,
2089
+ # :commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore
2090
+ # return
2091
+ # else
2092
+
2093
+
2094
+ # invoke :exec_remote, [cmd_to_exec.split(" ")], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
2095
+ # :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
2096
+ # :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore
2097
+ # return
2098
+ # end
1997
2099
  end
1998
2100
 
1999
2101
 
@@ -2013,6 +2115,26 @@ module Cnvrg
2013
2115
 
2014
2116
 
2015
2117
  def exec(*cmd)
2118
+ real_cmd = []
2119
+ args = [].replace(cmd)
2120
+ copy_args = [].replace(cmd)
2121
+ args.each_with_index do |p,i|
2122
+
2123
+ if !p.starts_with? "--" and !p.starts_with? "-"
2124
+ break
2125
+ else
2126
+ real_cmd << p
2127
+ if !p.include? "="
2128
+ if i+1< args.size
2129
+ real_cmd << args[i+1]
2130
+ copy_args.delete(args[i+1])
2131
+ end
2132
+ end
2133
+ copy_args.delete(p)
2134
+
2135
+ end
2136
+ end
2137
+
2016
2138
  log = []
2017
2139
  cpu_average =0
2018
2140
  memory_average = 0
@@ -2069,7 +2191,7 @@ module Cnvrg
2069
2191
  options_hash = Hash[options]
2070
2192
  options_hash.except!("image", "indocker")
2071
2193
  exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
2072
- command_to_run = cmd.join(" ")
2194
+ command_to_run = copy_args.join(" ")
2073
2195
  command = ["/bin/bash", "-lc", "cnvrg exec --indocker #{exec_options} #{command_to_run} #{exec_args}"]
2074
2196
  puts container.exec(command, tty: true)
2075
2197
  container.stop()
@@ -2077,7 +2199,7 @@ module Cnvrg
2077
2199
  end
2078
2200
  end
2079
2201
  start_commit = @project.last_local_commit
2080
- cmd = cmd.join("\s")
2202
+ cmd = copy_args.join("\s")
2081
2203
 
2082
2204
  say "Running: #{cmd}\n", Thor::Shell::Color::BLUE
2083
2205
 
@@ -2439,6 +2561,24 @@ module Cnvrg
2439
2561
 
2440
2562
 
2441
2563
  def exec_remote(*cmd)
2564
+ real_cmd = []
2565
+ args = cmd[0].split(" ")
2566
+ copy_args = cmd[0].split(" ")
2567
+ args.each_with_index do |p,i|
2568
+ if !p.starts_with? "--" and !p.starts_with? "-"
2569
+ break
2570
+ else
2571
+ real_cmd << p
2572
+ if !p.include? "="
2573
+ if i+1< args.size
2574
+ real_cmd << args[i+1]
2575
+ copy_args.delete(args[i+1])
2576
+ end
2577
+ end
2578
+ copy_args.delete(p)
2579
+
2580
+ end
2581
+ end
2442
2582
  verify_logged_in(true)
2443
2583
  log_start(__method__, args, options)
2444
2584
  working_dir = is_cnvrg_dir
@@ -2478,9 +2618,9 @@ module Cnvrg
2478
2618
  upload_output_option = "--upload_output=#{upload_output}"
2479
2619
  end
2480
2620
  options_hash = Hash[options]
2481
- options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit")
2621
+ options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "local","small","medium","large","gpu","gpuxl","gpuxxl")
2482
2622
  exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
2483
- command = "#{exec_options} #{upload_output_option} #{cmd.flatten.join(" ")}"
2623
+ command = "#{exec_options} #{upload_output_option} #{copy_args.flatten.join(" ")}"
2484
2624
  commit_to_run = options["commit"] || nil
2485
2625
  if !schedule.nil? and !schedule.empty?
2486
2626
 
@@ -4573,7 +4713,8 @@ module Cnvrg
4573
4713
  end
4574
4714
 
4575
4715
  def create_tar(path_in, path_out, tar_files)
4576
- `tar -czf #{path_out} -T #{tar_files} > /dev/null 2>&1`
4716
+ #The cd is meant for cases when running cnvrg data uplaod not in the main folder
4717
+ `cd #{path_in} && tar -czf #{path_out} -T #{tar_files} > /dev/null 2>&1`
4577
4718
  return $?.success?
4578
4719
  end
4579
4720
 
@@ -4583,6 +4724,61 @@ module Cnvrg
4583
4724
  end
4584
4725
  end
4585
4726
  end
4727
+ class Data < SubCommandBase
4728
+ desc "data init", "init data folder"
4729
+ method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
4730
+ def init
4731
+ cli = Cnvrg::CLI.new()
4732
+ public = options["public"]
4733
+ cli.init_data(public)
4734
+ end
4735
+ desc "data upload", "upload data folder"
4736
+ method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
4737
+ method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
4738
+ method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
4739
+ def upload
4740
+ cli = Cnvrg::CLI.new()
4741
+ ignore = options["ignore"]
4742
+ verbose = options["verbose"]
4743
+ sync = options["sync"]
4744
+
4745
+ cli.upload_data_tar(ignore, verbose,sync)
4746
+ end
4747
+ desc 'data download', 'pull data'
4748
+ method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
4749
+ method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
4750
+
4751
+ def download()
4752
+ cli = Cnvrg::CLI.new()
4753
+ verbose = options["verbose"]
4754
+ sync = options["sync"]
4755
+
4756
+ cli.download_data(verbose,sync,Dir.pwd)
4757
+
4758
+ end
4759
+ desc 'data clone', 'clone datset'
4760
+ def clone(dataset_url)
4761
+ cli = Cnvrg::CLI.new()
4762
+ cli.clone_data(dataset_url)
4763
+
4764
+ end
4765
+ desc 'data list', 'list of datasets'
4766
+ def list()
4767
+ cli = Cnvrg::CLI.new()
4768
+
4769
+ cli.list_dataset()
4770
+
4771
+ end
4772
+ desc 'data commits', 'pull data'
4773
+
4774
+ def commits()
4775
+ cli = Cnvrg::CLI.new()
4776
+ cli.list_dataset_commits()
4777
+
4778
+ end
4779
+
4780
+ end
4781
+
4586
4782
  end
4587
4783
 
4588
4784
 
data/lib/cnvrg/data.rb CHANGED
@@ -1,15 +1,15 @@
1
- require 'thor'
2
1
  require 'cnvrg/cli'
3
-
2
+ require 'thor'
4
3
  class SubCommandBase < Thor
5
- def self.banner(command, namespace = nil, subcommand = false)
6
- "#{basename} #{subcommand_prefix} #{command.usage}"
7
- end
4
+ def self.banner(command, namespace = nil, subcommand = false)
5
+ "#{basename} #{command.usage}"
6
+ end
7
+
8
+ def self.subcommand_prefix
9
+ self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) { |match| match[0].downcase }.gsub(%r{[A-Z]}) { |match| "-#{match[0].downcase}" }
10
+ end
11
+ end
8
12
 
9
- def self.subcommand_prefix
10
- self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) { |match| match[0].downcase }.gsub(%r{[A-Z]}) { |match| "-#{match[0].downcase}" }
11
- end
12
- end
13
13
  module Cnvrg
14
14
  class Data < SubCommandBase
15
15
  desc "data init", "init data folder"
data/lib/cnvrg/dataset.rb CHANGED
@@ -181,7 +181,7 @@ module Cnvrg
181
181
  end
182
182
  def url
183
183
  url = Cnvrg::Helpers.remote_url
184
- "#{url}/#{self.owner}/datsets/#{self.slug}"
184
+ "#{url}/#{self.owner}/datasets/#{self.slug}"
185
185
  end
186
186
  def generate_idx
187
187
  if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
@@ -206,12 +206,12 @@ module Cnvrg
206
206
  next
207
207
  end
208
208
  sha1 = Digest::SHA1.file(e).hexdigest
209
- if old_idx.nil? or old_idx.to_h[:tree].nil?
209
+ if old_idx.nil? or old_idx.to_h["tree"].nil?
210
210
  tree_idx[label] = {sha1: sha1, commit_time: nil}
211
- elsif old_idx[:tree][label].nil? or old_idx[:tree][label][:sha1] != sha1
211
+ elsif old_idx["tree"][label].nil? or old_idx["tree"][label]["sha1"] != sha1
212
212
  tree_idx[label] = {sha1: sha1, commit_time: nil}
213
213
  else
214
- tree_idx[label] = old_idx[:tree][label]
214
+ tree_idx[label] = old_idx["tree"][label]
215
215
  end
216
216
  end
217
217
  end
data/lib/cnvrg/version.rb CHANGED
@@ -1,4 +1,4 @@
1
1
  module Cnvrg
2
- VERSION = '0.0.1411'
2
+ VERSION = '0.0.1412'
3
3
  end
4
4
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cnvrg
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1411
4
+ version: 0.0.1412
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-05-24 00:00:00.000000000 Z
12
+ date: 2017-05-28 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: bundler