cnvrg 0.0.1412 → 0.0.1414

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: af467f0b591d7f5b4c48f1ea659ce6a8af10fa4a
-  data.tar.gz: a24c8338c54b12502ad9139f0fd12bd8865c5dc1
+  metadata.gz: a3a999e3ce7616d42f0a81033156a8d5819b68fd
+  data.tar.gz: 5dfc8054f9ee3f38cefaef9a06b113f72e3d12ba
 SHA512:
-  metadata.gz: 560278384f51b69ec1a01c629a9089e48d93b5c6630ae5e406f4de697e593cd6b7368c1634f7a66a8d0f0c6446823f3682121ac14fc936738b2d3b87d8dbbd22
-  data.tar.gz: 76a8f5b59fe26c179496105ee28efd6103ccf32abfb088759424c4bf52a15dc7b8a7934cf8f72cf8cd7e50323e9455dda0ac02b7283dcada0478a0eb71b2ac01
+  metadata.gz: cd4af46476dcb927770dd1c88c782ab9d2125e94cc496441f8b465b8edbcd22a2dd02d9d5dbb299c2aa1ac9fa183b2ece4c802f2041b243949a1bf385c317877
+  data.tar.gz: 2ec0f4e18e2571ce46dd77fa20bf7b6d5b5e667c4d76816502d5e86799c47417a0faec100c46d97566278fa4d68ce4779ead5e4e320ce767a6da711b3fc2d721

data/lib/cnvrg/cli.rb

@@ -55,6 +55,7 @@ class Thor
         # with pre-parsed options.
         array_options = []
         hash_options = local_options
+
       end
 
       # Let Thor::Options parse the options first, so it can remove
@@ -63,32 +64,36 @@ class Thor
       stop_on_unknown = Thor.stop_on_unknown_option? config[:current_command]
       opts = Thor::Options.new(parse_options, hash_options, stop_on_unknown)
       real_options = []
-      if args.empty?
-      real_args = [].replace(array_options)
 
-      array_options.each_with_index do |p,i|
-        if !p.starts_with? "--" and !p.starts_with? "-"
-          break
-        else
-          real_options << p
-          if !p.include? "="
-            if i+1< array_options.size
-              real_options << array_options[i+1]
-              real_args.delete(array_options[i+1])
+      real_args = [].replace(array_options)
+      if local_options.is_a? (Array) and !local_options.empty?
+        array_options.each_with_index do |p, i|
+          opt = p
+          if p.include? "="
+            opt = p.split("=")[0]
+          end
+          option = is_option(parse_options.values, opt)
+          if !option
+            break
+          else
+            real_options << p
+            real_args.delete(p)
+            if !p.include? "=" and option.type != :boolean
+              if i+1< array_options.size
+                real_options << array_options[i+1]
+                real_args.delete(array_options[i+1])
+              end
             end
+
           end
-          real_args.delete(p)
 
         end
-      end
-      args = real_args
-      else
-        if local_options.is_a? Array
-          args = [args + local_options].flatten
-        end
 
+        args = real_args
+      else
       end
 
+
       self.options = opts.parse(real_options)
       self.options = config[:class_options].merge(options) if config[:class_options]
 
@@ -101,19 +106,26 @@ class Thor
       # arguments declared using #argument (this is primarily used
       # by Thor::Group). Tis will leave us with the remaining
       # positional arguments.
-      to_parse  = args
+      to_parse = args
       to_parse += opts.remaining unless self.class.strict_args_position?(config)
       thor_args = Thor::Arguments.new(self.class.arguments)
       thor_args.parse(to_parse).each { |k, v| __send__("#{k}=", v) }
       @args = thor_args.remaining
     end
 
+    def is_option (options, p)
+      options.each do |o|
+        if (!o.aliases.nil? and o.aliases[0].split(",").include? p) or o.switch_name.eql? p
+          return o
+        end
+      end
+      return false
+    end
+
   end
 end
 
 
-
-
 module Cnvrg
 
   class CLI < Thor
@@ -235,61 +247,65 @@ module Cnvrg
     desc 'set_default_owner', 'set default owner'
 
     def set_default_owner
+      begin
+        path = File.expand_path('~')+"/.cnvrg/config.yml"
+        if !File.exist?(path)
+          say "Couldn't find ~/.cnvrg/config.yml file, please logout and login again", Thor::Shell::Color::RED
 
-      path = File.expand_path('~')+"/.cnvrg/config.yml"
-      if !File.exist?(path)
-        say "Couldn't find ~/.cnvrg/config.yml file, please logout and login again", Thor::Shell::Color::RED
-
-        exit(0)
-      end
-      config = YAML.load_file(path)
-
-      username = config.to_h[:username]
-      res = Cnvrg::API.request("/users/#{username}/get_possible_owners", 'GET')
-      if Cnvrg::CLI.is_response_success(res)
-        owner = username
-        result = res["result"]
-        owners = result["owners"]
-        urls = result["urls"]
-        choose_owner = result["username"]
-        if owners.empty?
-        else
-          owners << choose_owner
-          chosen = false
-          while !chosen
-            owners_id = owners.each_with_index.map { |x, i| "#{i+1}. #{x}" }
-            choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
-
-            if choose_owner =~ /[[:digit:]]/
-              ow_index = choose_owner.to_i-1
-              if ow_index<0 or ow_index >= owners.size
-                say "No such owner, please choose again", Thor::Shell::Color::BLUE
-                chosen = false
-                next
-              end
-              choose_owner = owners[choose_owner.to_i-1]
-              chosen = true
+          exit(0)
+        end
+        config = YAML.load_file(path)
 
-            else
+        username = config.to_h[:username]
+        res = Cnvrg::API.request("/users/#{username}/get_possible_owners", 'GET')
+        if Cnvrg::CLI.is_response_success(res)
+          owner = username
+          result = res["result"]
+          owners = result["owners"]
+          urls = result["urls"]
+          choose_owner = result["username"]
+          if owners.empty?
+          else
+            owners << choose_owner
+            chosen = false
+            while !chosen
+              owners_id = owners.each_with_index.map { |x, i| "#{i+1}. #{x}" }
+              choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
+
+              if choose_owner =~ /[[:digit:]]/
+                ow_index = choose_owner.to_i-1
+                if ow_index<0 or ow_index >= owners.size
+                  say "No such owner, please choose again", Thor::Shell::Color::BLUE
+                  chosen = false
+                  next
+                end
+                choose_owner = owners[choose_owner.to_i-1]
+                chosen = true
 
-              owners_lower = owners.map { |o| o.downcase }
-              ow_index = owners_lower.index(choose_owner.downcase)
-              if ow_index.nil?
-                say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
               else
-                chosen = true
+
+                owners_lower = owners.map { |o| o.downcase }
+                ow_index = owners_lower.index(choose_owner.downcase)
+                if ow_index.nil?
+                  say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
+                else
+                  chosen = true
+                end
               end
-            end
 
-          end
+            end
 
 
+          end
+          if set_owner(choose_owner, result["username"], urls[ow_index])
+            say "Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN
+          else
+            say "Setting default owenr has failed, try to run cnvrg --config-default-owner", Thor::Shell::Color::RED
+          end
         end
-        if set_owner(choose_owner, result["username"], urls[ow_index])
-          say "Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN
-        else
-          say "Setting default owenr has failed, try to run cnvrg --config-default-owner", Thor::Shell::Color::RED
-        end
+      rescue SignalException
+        say "\nAborting"
+        exit(1)
       end
     end
 
@@ -721,7 +737,7 @@ module Cnvrg
           say "Downloading data", Thor::Shell::Color::BLUE
 
 
-          download_data(false, false, path = working_dir,in_dir=false)
+          download_data(false, false, path = working_dir, in_dir=false)
 
 
           check = Helpers.checkmark
@@ -811,7 +827,7 @@ module Cnvrg
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
 
-    def download_data(verbose, sync, path=Dir.pwd,in_dir=true)
+    def download_data(verbose, sync, path=Dir.pwd, in_dir=true)
       begin
         verify_logged_in(in_dir)
         log_start(__method__, args, options)
@@ -1183,7 +1199,7 @@ module Cnvrg
 
 
               @files.rollback_commit(commit_sha1)
-              say "Rolling Back all changes.", Thor::Shell::Color::RED
+              say "Can't upload, Rolling Back all changes.", Thor::Shell::Color::RED
               exit(1)
             end
 
@@ -1193,7 +1209,7 @@ module Cnvrg
 
 
             @files.rollback_commit(commit_sha1)
-            say "Rolling Back all changes.", Thor::Shell::Color::RED
+            say "Can't upload, Rolling Back all changes.", Thor::Shell::Color::RED
             exit(1)
           end
 
@@ -2060,34 +2076,32 @@ module Cnvrg
       options_hash = Hash[options]
       real_options = []
       options_hash.each do |o|
-        real_options << o  if (!o[1].eql? "" and !["small","medium","large","gpu","gpuxl","gpuxxl"].include? o[0])
+        real_options << o if (!o[1].eql? "" and !["small", "medium", "large", "gpu", "gpuxl", "gpuxxl"].include? o[0])
       end
       if local
-        exec_options = real_options.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
-        cmd_to_exec = "#{exec_options} #{cmd.join(" ")}"
-        return exec(cmd_to_exec)
+        invoke :exec, cmd, :sync_before => sync_before, :sync_after => sync_after, :title => title,
+                 :log => log, :email_notification => email_notification, :upload_output => upload_output,
+                 :commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore
+          return
       else
-        real_options.delete(["local",false])
+        real_options.delete(["local", false])
         instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
-                                    "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
+                     "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
         instance_type = get_instance_type(instances)
         if !instance_type.nil? and !instance_type.empty?
-          real_options << ["machine_type",instance_type]
-        end
-        exec_options = real_options.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
-        cmd_to_exec = "#{exec_options} #{cmd.join(" ")}"
-        return exec_remote(cmd_to_exec)
-        # invoke :exec_remote, [cmd_to_exec], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
-        #          :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
-        #          :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore
-        #   return
+          real_options << ["machine_type", instance_type]
+        end
+        # exec_options = real_options.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
+        # cmd_to_exec = "#{exec_options} #{cmd.join(" ")}"
+        # return exec_remote(cmd_to_exec)
+        invoke :exec_remote, cmd, :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
+               :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
+               :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore
+        return
       end
 
       # if local
-      #   invoke :exec, [cmd_to_exec.split(" ")], :sync_before => sync_before, :sync_after => sync_after, :title => title,
-      #          :log => log, :email_notification => email_notification, :upload_output => upload_output,
-      #          :commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore
-      #   return
+      #
       # else
 
 
@@ -2100,41 +2114,21 @@ module Cnvrg
 
 
     desc '', '', :hide => true
-    method_option :sync_before, :type => :boolean, :aliases => ["-sb"], :default => true
-    method_option :sync_after, :type => :boolean, :aliases => ["-sa"], :default => true
+    method_option :sync_before, :type => :boolean, :aliases => ["-sb,--sync_before"], :default => true
+    method_option :sync_after, :type => :boolean, :aliases => ["-sa,--sync_after"], :default => true
     method_option :title, :type => :string, :aliases => ["-t"], :default => ""
     method_option :log, :type => :boolean, :aliases => ["-l", "--log"], :default => false
-    method_option :email_notification, :type => :boolean, :aliases => ["-en"], :default => false
-    method_option :upload_output, :type => :string, :aliases => ["-uo"], :default => ""
+    method_option :email_notification, :type => :boolean, :aliases => ["-en,--email_notification"], :default => false
+    method_option :upload_output, :type => :string, :aliases => ["-uo,--upload_output"], :default => ""
     method_option :commit, :type => :string, :aliases => ["-c"], :default => ""
     method_option :image, :type => :string, :aliases => ["-i"], :default => ""
-    method_option :indocker, :type => :boolean, :default => false
+    method_option :indocker, :type => :boolean, :aliases => ["--indocker"], :default => false
     method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
     method_option :data_commit, :type => :string, :aliases => ["-dc", "--data_commit"], :default => ""
     method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
 
 
     def exec(*cmd)
-      real_cmd = []
-      args = [].replace(cmd)
-      copy_args = [].replace(cmd)
-      args.each_with_index do |p,i|
-
-        if !p.starts_with? "--" and !p.starts_with? "-"
-          break
-        else
-          real_cmd << p
-          if !p.include? "="
-            if i+1< args.size
-              real_cmd << args[i+1]
-              copy_args.delete(args[i+1])
-            end
-          end
-          copy_args.delete(p)
-
-        end
-      end
-
       log = []
       cpu_average =0
       memory_average = 0
@@ -2199,7 +2193,7 @@ module Cnvrg
           end
         end
         start_commit = @project.last_local_commit
-        cmd = copy_args.join("\s")
+        cmd = cmd.join("\s")
 
         say "Running: #{cmd}\n", Thor::Shell::Color::BLUE
 
@@ -2561,24 +2555,6 @@ module Cnvrg
 
 
     def exec_remote(*cmd)
-      real_cmd = []
-      args = cmd[0].split(" ")
-      copy_args = cmd[0].split(" ")
-      args.each_with_index do |p,i|
-        if !p.starts_with? "--" and !p.starts_with? "-"
-          break
-        else
-          real_cmd << p
-          if !p.include? "="
-            if i+1< args.size
-              real_cmd << args[i+1]
-              copy_args.delete(args[i+1])
-            end
-          end
-          copy_args.delete(p)
-
-        end
-      end
       verify_logged_in(true)
       log_start(__method__, args, options)
       working_dir = is_cnvrg_dir
@@ -2588,7 +2564,6 @@ module Cnvrg
         grid = options["grid"] || nil
         data = options["data"] || nil
         data_commit = options["data_commit"] || nil
-        ignore = options[:ignore] || ""
 
         instance_type = options["machine_type"] || nil
         schedule = options["schedule"] || ""
@@ -2618,9 +2593,9 @@ module Cnvrg
           upload_output_option = "--upload_output=#{upload_output}"
         end
         options_hash = Hash[options]
-        options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "local","small","medium","large","gpu","gpuxl","gpuxxl")
+        options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl")
         exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
-        command = "#{exec_options}  #{upload_output_option} #{copy_args.flatten.join(" ")}"
+        command = "#{exec_options}  #{upload_output_option} #{cmd.flatten.join(" ")}"
         commit_to_run = options["commit"] || nil
         if !schedule.nil? and !schedule.empty?
 
@@ -2657,7 +2632,6 @@ module Cnvrg
 
         say "Running remote experiment", Thor::Shell::Color::BLUE
         exp = Experiment.new(project.owner, project.slug)
-
         res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid, path_to_cmd, data, data_commit)
         if Cnvrg::CLI.is_response_success(res)
           # if res["result"]["machine"] == -1
@@ -4727,23 +4701,27 @@ module Cnvrg
   class Data < SubCommandBase
     desc "data init", "init data folder"
     method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
+
     def init
       cli = Cnvrg::CLI.new()
       public = options["public"]
       cli.init_data(public)
     end
+
     desc "data upload", "upload data folder"
     method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+
     def upload
       cli = Cnvrg::CLI.new()
       ignore = options["ignore"]
       verbose = options["verbose"]
       sync = options["sync"]
 
-      cli.upload_data_tar(ignore, verbose,sync)
+      cli.upload_data_tar(ignore, verbose, sync)
     end
+
     desc 'data download', 'pull data'
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
@@ -4753,22 +4731,27 @@ module Cnvrg
       verbose = options["verbose"]
       sync = options["sync"]
 
-      cli.download_data(verbose,sync,Dir.pwd)
+      cli.download_data(verbose, sync, Dir.pwd)
 
     end
+
     desc 'data clone', 'clone datset'
+
     def clone(dataset_url)
       cli = Cnvrg::CLI.new()
       cli.clone_data(dataset_url)
 
     end
+
     desc 'data list', 'list of datasets'
+
     def list()
       cli = Cnvrg::CLI.new()
 
       cli.list_dataset()
 
     end
+
     desc 'data commits', 'pull data'
 
     def commits()

data/lib/cnvrg/datafiles.rb

@@ -249,12 +249,12 @@ module Cnvrg
 
           s4cmd=`pip freeze |grep s4cmd > /dev/null 2>&1` ;  s4cmd_suc=$?.success?
           if !s4cmd_suc
-            `pip install s4cmd  > /dev/null 2>&1`
+            `pip install s4cmd  > /dev/null 2>&1` ; s4cmd_install_suc=$?.success?
           end
 
         end
 
-        if !is_python
+        if !s4cmd_suc or !s4cmd_install_suc
           s3 = Aws::S3::Resource.new(
               :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
               :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
@@ -303,7 +303,7 @@ module Cnvrg
 
         return resp
 
-      rescue =>e
+      rescue
         if File.exist? tmp
           FileUtils.rm_rf [tmp]
         end

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-    VERSION = '0.0.1412'
+    VERSION = '0.0.1414'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.0.1412
+  version: 0.0.1414
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-28 00:00:00.000000000 Z
+date: 2017-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -359,9 +359,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.3
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.
 test_files: []
-has_rdoc: