cnvrg 0.0.1411 → 0.0.1412

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 093f03d38281b47f7a3f251b5875aa89ac27a39c
-  data.tar.gz: 45a9001d2a87bdae1f7e097975a56b2c3b57be9d
+  metadata.gz: af467f0b591d7f5b4c48f1ea659ce6a8af10fa4a
+  data.tar.gz: a24c8338c54b12502ad9139f0fd12bd8865c5dc1
 SHA512:
-  metadata.gz: 50991f6a61b5686e956dcd12baa0ca2bd1bba5ed826c73181144951949a6cda3a2e2dbceb75111cc26702c1b99ade85f74e640dc53dc3df46e28a0cbfb893ed9
-  data.tar.gz: 153369319301d57aecbd07022ea224f7b2cadb6a56bb1365f9a42a5fa86bf0b8cc3593de05eb6bea0ade20dd0fbd50820a7313439ab7678063ab5a9437385897
+  metadata.gz: 560278384f51b69ec1a01c629a9089e48d93b5c6630ae5e406f4de697e593cd6b7368c1634f7a66a8d0f0c6446823f3682121ac14fc936738b2d3b87d8dbbd22
+  data.tar.gz: 76a8f5b59fe26c179496105ee28efd6103ccf32abfb088759424c4bf52a15dc7b8a7934cf8f72cf8cd7e50323e9455dda0ac02b7283dcada0478a0eb71b2ac01

data/lib/cnvrg/cli.rb

@@ -20,6 +20,7 @@ require 'cnvrg/experiment'
 require 'cnvrg/Images'
 require 'cnvrg/dataset'
 require 'cnvrg/datafiles'
+require 'cnvrg/data'
 require 'etc'
 require 'logstash-logger'
 require 'cnvrg/job'
@@ -32,12 +33,87 @@ require 'zip'
 require 'active_support/all'
 require 'thor'
 require 'pathname'
-require 'cnvrg/data'
-# DEV VERSION
 
-# include Cnvrg::Tar
 
-# include Tar
+class Thor
+  module Base
+    def initialize(args = [], local_options = {}, config = {})
+      parse_options = Thor.class_options
+
+      # The start method splits inbound arguments at the first argument
+      # that looks like an option (starts with - or --). It then calls
+      # new, passing in the two halves of the arguments Array as the
+      # first two parameters.
+
+      command_options = config.delete(:command_options) # hook for start
+      parse_options = parse_options.merge(command_options) if command_options
+      if local_options.is_a?(Array)
+        array_options = local_options
+        hash_options = {}
+      else
+        # Handle the case where the class was explicitly instantiated
+        # with pre-parsed options.
+        array_options = []
+        hash_options = local_options
+      end
+
+      # Let Thor::Options parse the options first, so it can remove
+      # declared options from the array. This will leave us with
+      # a list of arguments that weren't declared.
+      stop_on_unknown = Thor.stop_on_unknown_option? config[:current_command]
+      opts = Thor::Options.new(parse_options, hash_options, stop_on_unknown)
+      real_options = []
+      if args.empty?
+      real_args = [].replace(array_options)
+
+      array_options.each_with_index do |p,i|
+        if !p.starts_with? "--" and !p.starts_with? "-"
+          break
+        else
+          real_options << p
+          if !p.include? "="
+            if i+1< array_options.size
+              real_options << array_options[i+1]
+              real_args.delete(array_options[i+1])
+            end
+          end
+          real_args.delete(p)
+
+        end
+      end
+      args = real_args
+      else
+        if local_options.is_a? Array
+          args = [args + local_options].flatten
+        end
+
+      end
+
+      self.options = opts.parse(real_options)
+      self.options = config[:class_options].merge(options) if config[:class_options]
+
+      # If unknown options are disallowed, make sure that none of the
+      # remaining arguments looks like an option.
+      opts.check_unknown! if Thor.check_unknown_options?(config)
+
+      # Add the remaining arguments from the options parser to the
+      # arguments passed in to initialize. Then remove any positional
+      # arguments declared using #argument (this is primarily used
+      # by Thor::Group). Tis will leave us with the remaining
+      # positional arguments.
+      to_parse  = args
+      to_parse += opts.remaining unless self.class.strict_args_position?(config)
+      thor_args = Thor::Arguments.new(self.class.arguments)
+      thor_args.parse(to_parse).each { |k, v| __send__("#{k}=", v) }
+      @args = thor_args.remaining
+    end
+
+  end
+end
+
+
+
+
 module Cnvrg
 
   class CLI < Thor
@@ -337,7 +413,6 @@ module Cnvrg
 
     def me()
       begin
-
         verify_logged_in(false)
         log_start(__method__, args, options)
         auth = Cnvrg::Auth.new
@@ -348,6 +423,7 @@ module Cnvrg
         end
 
         log_end(0)
+
       rescue SignalException
         log_end(-1)
 
@@ -593,7 +669,7 @@ module Cnvrg
           url = @dataset.url
           check = Helpers.checkmark
           say "#{check} Link finished successfully", Thor::Shell::Color::GREEN
-          # say "#{dataset_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN
+          say "#{dataset_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN
           log_end(0)
 
         else
@@ -645,7 +721,7 @@ module Cnvrg
           say "Downloading data", Thor::Shell::Color::BLUE
 
 
-          download_data(false, false, path = working_dir)
+          download_data(false, false, path = working_dir,in_dir=false)
 
 
           check = Helpers.checkmark
@@ -735,9 +811,9 @@ module Cnvrg
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
 
-    def download_data(verbose, sync, path=Dir.pwd)
+    def download_data(verbose, sync, path=Dir.pwd,in_dir=true)
       begin
-        verify_logged_in(true)
+        verify_logged_in(in_dir)
         log_start(__method__, args, options)
         if path.nil? or path.empty?
           path = Dir.pwd
@@ -1082,13 +1158,15 @@ module Cnvrg
           home_dir = File.expand_path('~')
           tar_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.tar.gz"
           tar_files_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.txt"
-          tar_files = (result["added"] + result["updated_on_local"] +["\n"]).join("\n")
+          tar_files = (result["added"] + result["updated_on_local"]).join("\n")
           File.open(tar_files_path, 'w') { |f| f.write tar_files }
           is_tar = create_tar(dataset_dir, tar_path, tar_files_path)
           if !is_tar
             say "ERROR: Couldn't compress data", Thor::Shell::Color::RED
             FileUtils.rm_rf([tar_path]) if File.exist? tar_path
             FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
+            @files.rollback_commit(commit_sha1)
+            say "Rolling Back all changes.", Thor::Shell::Color::RED
             exit(1)
           end
           say "Uploading data", Thor::Shell::Color::BLUE
@@ -1194,7 +1272,7 @@ module Cnvrg
 
     end
 
-    desc 'list data', 'List all dataset you currently have'
+    desc 'data list', 'List all dataset you currently have'
 
     def list_dataset
       verify_logged_in(false)
@@ -1212,7 +1290,7 @@ module Cnvrg
 
     end
 
-    desc 'list data commits', 'List all commits for a specific dataset'
+    desc 'data commits', 'List all commits for a specific dataset'
 
     def list_dataset_commits()
       verify_logged_in(true)
@@ -1952,7 +2030,7 @@ module Cnvrg
     method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
     method_option :sync_before, :type => :boolean, :aliases => ["-sb"], :default => true
     method_option :sync_after, :type => :boolean, :aliases => ["-sa"], :default => true
-    method_option :title, :type => :string, :aliases => ["-t"], :default => ""
+    method_option :title, :type => :string, :aliases => ["-t", "--title"], :default => ""
     method_option :log, :type => :boolean, :aliases => ["--log"], :default => false
     method_option :email_notification, :type => :boolean, :aliases => ["-en"], :default => false
     method_option :upload_output, :type => :string, :aliases => ["-uo"], :default => ""
@@ -1979,21 +2057,45 @@ module Cnvrg
       data = options["data"]
       data_commit = options["data_commit"]
       ignore = options["ignore"]
-
+      options_hash = Hash[options]
+      real_options = []
+      options_hash.each do |o|
+        real_options << o  if (!o[1].eql? "" and !["small","medium","large","gpu","gpuxl","gpuxxl"].include? o[0])
+      end
       if local
-        invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
-               :log => log, :email_notification => email_notification, :upload_output => upload_output,
-               :commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore
-        return
+        exec_options = real_options.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
+        cmd_to_exec = "#{exec_options} #{cmd.join(" ")}"
+        return exec(cmd_to_exec)
       else
+        real_options.delete(["local",false])
         instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
-                     "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
+                                    "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
         instance_type = get_instance_type(instances)
-        invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
-               :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
-               :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore
-        return
+        if !instance_type.nil? and !instance_type.empty?
+          real_options << ["machine_type",instance_type]
+        end
+        exec_options = real_options.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
+        cmd_to_exec = "#{exec_options} #{cmd.join(" ")}"
+        return exec_remote(cmd_to_exec)
+        # invoke :exec_remote, [cmd_to_exec], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
+        #          :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
+        #          :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore
+        #   return
       end
+
+      # if local
+      #   invoke :exec, [cmd_to_exec.split(" ")], :sync_before => sync_before, :sync_after => sync_after, :title => title,
+      #          :log => log, :email_notification => email_notification, :upload_output => upload_output,
+      #          :commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore
+      #   return
+      # else
+
+
+      #   invoke :exec_remote, [cmd_to_exec.split(" ")], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
+      #          :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
+      #          :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore
+      #   return
+      # end
     end
 
 
@@ -2013,6 +2115,26 @@ module Cnvrg
 
 
     def exec(*cmd)
+      real_cmd = []
+      args = [].replace(cmd)
+      copy_args = [].replace(cmd)
+      args.each_with_index do |p,i|
+
+        if !p.starts_with? "--" and !p.starts_with? "-"
+          break
+        else
+          real_cmd << p
+          if !p.include? "="
+            if i+1< args.size
+              real_cmd << args[i+1]
+              copy_args.delete(args[i+1])
+            end
+          end
+          copy_args.delete(p)
+
+        end
+      end
+
       log = []
       cpu_average =0
       memory_average = 0
@@ -2069,7 +2191,7 @@ module Cnvrg
             options_hash = Hash[options]
             options_hash.except!("image", "indocker")
             exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
-            command_to_run = cmd.join(" ")
+            command_to_run = copy_args.join(" ")
             command = ["/bin/bash", "-lc", "cnvrg exec --indocker #{exec_options} #{command_to_run} #{exec_args}"]
             puts container.exec(command, tty: true)
             container.stop()
@@ -2077,7 +2199,7 @@ module Cnvrg
           end
         end
         start_commit = @project.last_local_commit
-        cmd = cmd.join("\s")
+        cmd = copy_args.join("\s")
 
         say "Running: #{cmd}\n", Thor::Shell::Color::BLUE
 
@@ -2439,6 +2561,24 @@ module Cnvrg
 
 
     def exec_remote(*cmd)
+      real_cmd = []
+      args = cmd[0].split(" ")
+      copy_args = cmd[0].split(" ")
+      args.each_with_index do |p,i|
+        if !p.starts_with? "--" and !p.starts_with? "-"
+          break
+        else
+          real_cmd << p
+          if !p.include? "="
+            if i+1< args.size
+              real_cmd << args[i+1]
+              copy_args.delete(args[i+1])
+            end
+          end
+          copy_args.delete(p)
+
+        end
+      end
       verify_logged_in(true)
       log_start(__method__, args, options)
       working_dir = is_cnvrg_dir
@@ -2478,9 +2618,9 @@ module Cnvrg
           upload_output_option = "--upload_output=#{upload_output}"
         end
         options_hash = Hash[options]
-        options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit")
+        options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "local","small","medium","large","gpu","gpuxl","gpuxxl")
         exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
-        command = "#{exec_options}  #{upload_output_option} #{cmd.flatten.join(" ")}"
+        command = "#{exec_options}  #{upload_output_option} #{copy_args.flatten.join(" ")}"
         commit_to_run = options["commit"] || nil
         if !schedule.nil? and !schedule.empty?
 
@@ -4573,7 +4713,8 @@ module Cnvrg
       end
 
       def create_tar(path_in, path_out, tar_files)
-        `tar -czf #{path_out} -T #{tar_files}  > /dev/null 2>&1`
+        #The cd is meant for cases when running cnvrg data uplaod not in the main folder
+        `cd #{path_in} && tar -czf #{path_out} -T #{tar_files}  > /dev/null 2>&1`
         return $?.success?
       end
 
@@ -4583,6 +4724,61 @@ module Cnvrg
       end
     end
   end
+  class Data < SubCommandBase
+    desc "data init", "init data folder"
+    method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
+    def init
+      cli = Cnvrg::CLI.new()
+      public = options["public"]
+      cli.init_data(public)
+    end
+    desc "data upload", "upload data folder"
+    method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+    def upload
+      cli = Cnvrg::CLI.new()
+      ignore = options["ignore"]
+      verbose = options["verbose"]
+      sync = options["sync"]
+
+      cli.upload_data_tar(ignore, verbose,sync)
+    end
+    desc 'data download', 'pull data'
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+
+    def download()
+      cli = Cnvrg::CLI.new()
+      verbose = options["verbose"]
+      sync = options["sync"]
+
+      cli.download_data(verbose,sync,Dir.pwd)
+
+    end
+    desc 'data clone', 'clone datset'
+    def clone(dataset_url)
+      cli = Cnvrg::CLI.new()
+      cli.clone_data(dataset_url)
+
+    end
+    desc 'data list', 'list of datasets'
+    def list()
+      cli = Cnvrg::CLI.new()
+
+      cli.list_dataset()
+
+    end
+    desc 'data commits', 'pull data'
+
+    def commits()
+      cli = Cnvrg::CLI.new()
+      cli.list_dataset_commits()
+
+    end
+
+  end
+
 end
 
 

data/lib/cnvrg/data.rb

@@ -1,15 +1,15 @@
-require 'thor'
 require 'cnvrg/cli'
-
+require 'thor'
 class SubCommandBase < Thor
-  def self.banner(command, namespace = nil, subcommand = false)
-    "#{basename} #{subcommand_prefix} #{command.usage}"
-  end
+ def self.banner(command, namespace = nil, subcommand = false)
+  "#{basename}  #{command.usage}"
+ end
+
+ def self.subcommand_prefix
+  self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) { |match| match[0].downcase }.gsub(%r{[A-Z]}) { |match| "-#{match[0].downcase}" }
+ end
+end
 
-  def self.subcommand_prefix
-    self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) { |match| match[0].downcase }.gsub(%r{[A-Z]}) { |match| "-#{match[0].downcase}" }
-  end
-  end
 module Cnvrg
   class Data < SubCommandBase
     desc "data init", "init data folder"

data/lib/cnvrg/dataset.rb

@@ -181,7 +181,7 @@ module Cnvrg
       end
     def url
       url = Cnvrg::Helpers.remote_url
-      "#{url}/#{self.owner}/datsets/#{self.slug}"
+      "#{url}/#{self.owner}/datasets/#{self.slug}"
     end
     def generate_idx
       if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
@@ -206,12 +206,12 @@ module Cnvrg
             next
           end
           sha1 = Digest::SHA1.file(e).hexdigest
-          if old_idx.nil? or old_idx.to_h[:tree].nil?
+          if old_idx.nil? or old_idx.to_h["tree"].nil?
             tree_idx[label] = {sha1: sha1, commit_time: nil}
-          elsif old_idx[:tree][label].nil? or old_idx[:tree][label][:sha1] != sha1
+          elsif old_idx["tree"][label].nil? or old_idx["tree"][label]["sha1"] != sha1
             tree_idx[label] = {sha1: sha1, commit_time: nil}
           else
-            tree_idx[label] = old_idx[:tree][label]
+            tree_idx[label] = old_idx["tree"][label]
           end
         end
       end

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-    VERSION = '0.0.1411'
+    VERSION = '0.0.1412'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.0.1411
+  version: 0.0.1412
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-24 00:00:00.000000000 Z
+date: 2017-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler