cnvrg 0.0.142561 → 0.0.142562

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d2a92149e9b565661704f3f91438b28c89976475
-  data.tar.gz: 9f5c2dc4a71823c9b5657fd941361e33305d06f6
+  metadata.gz: 965ed0f54a906aa040983dd7857810a934e6ddf8
+  data.tar.gz: 6ec1122320418d0f0f45d21be77607d3751eb9f5
 SHA512:
-  metadata.gz: 0aace663fb6460ade819c6676fb96b11cbbb286340e0b2599b6be6bb6d1ae5f3e6098270b9f37976cae7ec743b6381f46dd1e1054cb4f56e71d249cbb875f43b
-  data.tar.gz: 47dd9900f83b208196690862ceba2511643fbf84fc2d1daa6a23b66c51c63279aa9766bcaee9811b72af9b4f44e605e7fa34e5b0e450bcbe67f4e62a7480c173
+  metadata.gz: 55aa453ff5c88e47c2b189927c6aa99da7c6864e3acafd0d319db8e7cff88db4e3411eead1c26e3cbbcf7b0765d1a82c0ed8a46cf9fb2ea674b2e7acb071a663
+  data.tar.gz: 39cc6c68eca8f1eefe319595bc0bca4bd3d9bd77f6c3a1034bf6c065e20b097e8f5ae8cc194a5f8d85fb56fca1dfa90946f8c2c7b00a38253843cdc09db326f5

data/cnvrg.gemspec

@@ -40,6 +40,8 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'rubyzip', '~> 1.2'
     spec.add_runtime_dependency 'activesupport', '~> 5.0'
     spec.add_runtime_dependency 'ruby-progressbar'
+    spec.add_runtime_dependency 'net-ssh'
+
 
 
     

data/lib/cnvrg.rb

@@ -3,6 +3,5 @@ require 'cnvrg/version'
 require 'cnvrg/cli'
 require 'thor'
 
-
 module Cnvrg
 end

data/lib/cnvrg/Images.rb

@@ -1,6 +1,8 @@
 require 'fileutils'
 require 'cnvrg/files'
 require 'docker'
+require 'net/ssh'
+
 
 module Cnvrg
   class Images
@@ -93,6 +95,73 @@ module Cnvrg
 
 
     end
+    def self.create_new_custom_image(type,owner,image_name,is_public,is_base,image_extend,python3)
+      response = Cnvrg::API.request("users/#{owner}/images/custom", 'POST', {instance_type:type,image_name:image_name,is_public:is_public,
+                                                                             is_base:is_base,image_extend:image_extend,
+                                                                             python3:python3})
+      return response
+    end
+    def self.revoke_custom_new_image(owner,slug)
+      response = Cnvrg::API.request("users/#{owner}/images/#{slug}/revoke_image", 'GET')
+      return response
+    end
+    def self.commit_custom_image(owner,slug)
+      response = Cnvrg::API.request("users/#{owner}/images/#{slug}/commit_custom_image", 'GET')
+      return response
+    end
+    def self.ssh_to_machine(resp)
+
+      sts_path = resp["result"]["sts_path"]
+
+      uri = URI.parse(sts_path)
+
+      http_object = Net::HTTP.new(uri.host, uri.port)
+      http_object.use_ssl = true if uri.scheme == 'https'
+      request = Net::HTTP::Get.new(sts_path)
+
+      body = ""
+      http_object.start do |http|
+        response = http.request request
+        body = response.read_body
+      end
+
+      URLcrypt::key = [body].pack('H*')
+
+      ip = URLcrypt.decrypt(resp["result"]["machine_i"])
+
+      user = URLcrypt.decrypt(resp["result"]["machine_u"])
+      key = URLcrypt.decrypt(resp["result"]["machine_k"])
+      tempssh = Tempfile.new "sshkey"
+      tempssh.write open(key).read
+      tempssh.rewind
+      key_path = tempssh.path
+      count = 0
+      while count < 5
+
+      begin
+          ssh = Net::SSH.start(ip, user=user, :keys => key_path, :timeout => 10)
+          if !ssh.nil?
+            return ssh
+          else
+            count+=1
+            sleep(2)
+
+          end
+        rescue
+      count+=1
+      sleep(2)
+
+
+      end
+      end
+      if tempssh
+        tempssh.close
+        tempssh.unlink
+      end
+      return false
+    end
+
+
 
     def create_custom_image(new_image_name,working_dir,stored_commands)
 

data/lib/cnvrg/api.rb

@@ -57,7 +57,10 @@ module Cnvrg
                 else
                     response
                 end
-            when 'POST'
+              when 'POST'
+                conn.options.timeout = 420
+                conn.options.open_timeout =420
+
                 response = conn.post "#{endpoint_uri}/#{resource}", data
 
                 if parse_request == true
@@ -70,12 +73,12 @@ module Cnvrg
                     fr.headers['Auth-Token'] = @pass
                     fr.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
                     fr.headers["Content-Type"] = "multipart/form-data"
-                    
+
                     fr.request :multipart
                     fr.request :url_encoded
                     fr.adapter :net_http
                 end
-                
+
 
                 # what if windows?
                 # data[:file] = Faraday::UploadIO.new(data[:absolute_path], content_type)
@@ -104,6 +107,7 @@ module Cnvrg
             else
             end
             rescue => e
+              puts e
                return nil
             end
 

data/lib/cnvrg/cli.rb

@@ -21,6 +21,7 @@ require 'cnvrg/Images'
 require 'cnvrg/dataset'
 require 'cnvrg/datafiles'
 require 'cnvrg/data'
+require 'cnvrg/ssh'
 require 'etc'
 require 'logstash-logger'
 require 'cnvrg/job'
@@ -1154,8 +1155,9 @@ module Cnvrg
     method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+    method_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
 
-    def upload_data_tar(ignore, verbose, sync)
+    def upload_data_tar(ignore, verbose, sync,no_compression)
 
       begin
         verify_logged_in(true)
@@ -1225,7 +1227,7 @@ module Cnvrg
           tar_files_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.txt"
           tar_files = (result["added"] + result["updated_on_local"]).join("\n")
           File.open(tar_files_path, 'w') { |f| f.write tar_files }
-          is_tar = create_tar(dataset_dir, tar_path, tar_files_path)
+          is_tar = create_tar(dataset_dir, tar_path, tar_files_path,no_compression)
           if !is_tar
             say "ERROR: Couldn't compress data", Thor::Shell::Color::RED
             FileUtils.rm_rf([tar_path]) if File.exist? tar_path
@@ -3541,7 +3543,6 @@ module Cnvrg
           container.stop()
         end
       rescue SignalException
-        log_End(-1)
         if container
           container.stop()
         end
@@ -3551,6 +3552,98 @@ module Cnvrg
 
     end
 
+    method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
+    method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
+    method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
+    method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
+    method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
+    method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
+    method_option :image, :type => :string, :aliases => ["-i","--image"], :default => ""
+    method_option :public, :type => :boolean, :aliases => ["-p","--public"], :default => false
+    method_option :base, :type => :boolean, :aliases => ["-b","--base"], :default => false
+    method_option :python3, :type => :boolean, :aliases => ["--python3"], :default => false
+
+    desc 'create_custom_image', 'run commands inside containers', :hide=>true
+
+    def create_custom_image(image_name)
+      begin
+      verify_logged_in(false)
+      log_start(__method__, args, options)
+      instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
+                   "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
+      instance_type = get_instance_type(instances)
+      image_extend = options["image"]
+      public = options["public"]
+      base = options["base"]
+      python3 = options["python3"]
+      owner = CLI.get_owner
+      checks = Helpers.checkmark()
+
+      say "Creating machine for custom image, this may take a few moments...", Thor::Shell::Color::BLUE
+
+      resp = Images.create_new_custom_image(instance_type,owner,image_name,public,base,image_extend,python3)
+      if Cnvrg::CLI.is_response_success(resp,false)
+        image_slug = resp["result"]["slug"]
+        container = resp["result"]["machine_c"]
+        say "#{checks} Created image and machine successfully", Thor::Shell::Color::GREEN
+        say "Connecting to machine", Thor::Shell::Color::BLUE
+        ssh = Ssh.new(resp)
+        if !ssh.is_ssh
+          say "Couldn't connect to machine,aborting", Thor::Shell::Color::RED
+          Images.revoke_custom_new_image(owner, image_slug)
+          log_end(-1,"Couldn't connect to machine,aborting")
+        end
+        say "run command until ctrl + c initiates", Thor::Shell::Color::BLUE
+        begin
+
+          while true
+            command = ask("$>")
+            puts ssh.exec_command(command)
+          end
+
+        rescue SignalException
+          say "Commiting Image..", Thor::Shell::Color::BLUE
+
+        end
+        resp = Images.commit_custom_image(owner,image_slug)
+        if Cnvrg::CLI.is_response_success(resp,false)
+          say "#{checks} Image commited successfuly, email will be sent when image is ready", Thor::Shell::Color::GREEN
+        end
+        if ssh
+          ssh.close_ssh()
+        end
+
+
+
+
+
+
+
+      end
+      rescue =>e
+        puts e
+        if image_slug
+          Images.revoke_custom_new_image(owner, image_slug)
+        end
+        if ssh
+          ssh.close_ssh()
+        end
+
+
+      rescue SignalException
+        if image_slug
+          Images.revoke_custom_new_image(owner, image_slug)
+        end
+        if ssh
+          ssh.close_ssh
+        end
+        say "\nAborting"
+        exit(1)
+      end
+
+    end
+
+
     desc 'build', 'run commands inside containers'
     method_option :install, :type => :string, :aliases => ["--i"], :default => nil, :desc => "Install from the given instructions file"
 
@@ -4952,10 +5045,14 @@ module Cnvrg
         end
       end
 
-      def create_tar(path_in, path_out, tar_files)
+      def create_tar(path_in, path_out, tar_files,no_compression=false)
         #The cd is meant for cases when running cnvrg data uplaod not in the main folder
         begin
-        `cd #{path_in} && tar -czf #{path_out} -T #{tar_files}`
+          if no_compression
+            `cd #{path_in} && tar -cf #{path_out} -T #{tar_files}`
+          else
+            `cd #{path_in} && tar -czf #{path_out} -T #{tar_files}`
+          end
         rescue => e
           puts "Exception while compressing data: #{e.message}"
         end

data/lib/cnvrg/data.rb

@@ -11,7 +11,9 @@ class SubCommandBase < Thor
 end
 
 module Cnvrg
+
   class Data < SubCommandBase
+    class_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
     desc "data init", "init data folder"
     method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
     def init
@@ -23,13 +25,15 @@ module Cnvrg
     method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+    method_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
+
     def upload
       cli = Cnvrg::CLI.new()
       ignore = options["ignore"]
       verbose = options["verbose"]
       sync = options["sync"]
-
-      cli.upload_data_tar(ignore, verbose,sync)
+      no_compression = options["no_compression"]
+      cli.upload_data_tar(ignore, verbose,sync,no_compression)
     end
     desc 'data download', 'pull data'
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false

data/lib/cnvrg/datafiles.rb

@@ -234,7 +234,7 @@ module Cnvrg
     def   upload_large_files_s3(upload_resp, file_path)
       begin
         sts_path = upload_resp["result"]["path_sts"]
-        s4cmd_path = upload_resp["result"]["path_s4cmd"]
+        # s4cmd_path = upload_resp["result"]["path_s4cmd"]
 
         uri = URI.parse(sts_path)
         http_object = Net::HTTP.new(uri.host, uri.port)
@@ -266,46 +266,45 @@ module Cnvrg
           s3 = Aws::S3::Resource.new(
               :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
               :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
-              :session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
               :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
           resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
               object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
               upload_file(file_path,{:use_accelerate_endpoint=>true})
 
-        else
-
-          s4cmd_uri = URI.parse(s4cmd_path)
-          s4cmd_http_object = Net::HTTP.new(s4cmd_uri.host, s4cmd_uri.port)
-          s4cmd_http_object.use_ssl = true if s4cmd_uri.scheme == 'https'
-          s4cmd_request = Net::HTTP::Get.new(s4cmd_path)
-
-          s4cmd_body = ""
-          s4cmd_http_object.start do |http|
-            response = http.request s4cmd_request
-            s4cmd_body = response.read_body
-          end
-
-          s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
-                                      aws_access_key_id=aws_access_key_id,
-                                      aws_secret_access_key=aws_secret_access_key)"," self.client = self.boto3.client('s3',
-                                      aws_access_key_id='#{ URLcrypt.decrypt(upload_resp["result"]["sts_a"])}',
-                                      aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["sts_s"])}',
-                                      aws_session_token='#{URLcrypt.decrypt(upload_resp["result"]["sts_st"])}')")
-
-          tmp = Tempfile.new('s4cmd.py')
-          tmp << s4cmd_new_body
-          tmp.flush
-          tmp.close
-
-            is_success = false
-          count = 0
-          while !is_success and count <3
-            resp = `python #{tmp.path} --num-threads=8 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)}  > /dev/null 2>&1`
-            is_success =$?.success?
-            count +=1
-
-          end
-          resp= is_success
+        # else
+
+          # s4cmd_uri = URI.parse(s4cmd_path)
+          # s4cmd_http_object = Net::HTTP.new(s4cmd_uri.host, s4cmd_uri.port)
+          # s4cmd_http_object.use_ssl = true if s4cmd_uri.scheme == 'https'
+          # s4cmd_request = Net::HTTP::Get.new(s4cmd_path)
+          #
+          # s4cmd_body = ""
+          # s4cmd_http_object.start do |http|
+          #   response = http.request s4cmd_request
+          #   s4cmd_body = response.read_body
+          # end
+          #
+          # s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
+          #                             aws_access_key_id=aws_access_key_id,
+          #                             aws_secret_access_key=aws_secret_access_key)"," self.client = self.boto3.client('s3',
+          #                             aws_access_key_id='#{ URLcrypt.decrypt(upload_resp["result"]["sts_a"])}',
+          #                             aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["sts_s"])}',
+          #                             aws_session_token='#{URLcrypt.decrypt(upload_resp["result"]["sts_st"])}')")
+          #
+          # tmp = Tempfile.new('s4cmd.py')
+          # tmp << s4cmd_new_body
+          # tmp.flush
+          # tmp.close
+          #
+          #   is_success = false
+          # count = 0
+          # while !is_success and count <3
+          #   resp = `python #{tmp.path} --num-threads=8 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)}  > /dev/null 2>&1`
+          #   is_success =$?.success?
+          #   count +=1
+          #
+          # end
+          # resp= is_success
 
         end
 

data/lib/cnvrg/ssh.rb

@@ -0,0 +1,95 @@
+require 'fileutils'
+require 'cnvrg/files'
+require 'docker'
+require 'net/ssh'
+
+
+module Cnvrg
+  class Ssh
+    attr_reader :is_ssh
+
+
+    def initialize(resp)
+      begin
+        @is_ssh = false
+        sts_path = resp["result"]["sts_path"]
+
+        uri = URI.parse(sts_path)
+
+        http_object = Net::HTTP.new(uri.host, uri.port)
+        http_object.use_ssl = true if uri.scheme == 'https'
+        request = Net::HTTP::Get.new(sts_path)
+
+        body = ""
+        http_object.start do |http|
+          response = http.request request
+          body = response.read_body
+        end
+
+        URLcrypt::key = [body].pack('H*')
+        ip = URLcrypt.decrypt(resp["result"]["machine_i"])
+
+        @user = URLcrypt.decrypt(resp["result"]["machine_u"])
+        key = URLcrypt.decrypt(resp["result"]["machine_k"])
+        @container = URLcrypt.decrypt(resp["result"]["machine_c"])
+
+        tempssh = Tempfile.new "sshkey"
+        tempssh.write open(key).read
+        tempssh.rewind
+        key_path = tempssh.path
+        count = 0
+        while count < 5
+
+          begin
+            @ssh = Net::SSH.start(ip, user=@user, :keys => key_path, :timeout => 10)
+            if !@ssh.nil?
+              @is_ssh = true
+              return
+            else
+              count+=1
+              sleep(2)
+
+            end
+          rescue
+            count+=1
+            sleep(2)
+
+
+          end
+        end
+        if tempssh
+          tempssh.close
+          tempssh.unlink
+        end
+        return false
+      rescue => e
+
+        puts e
+      end
+    end
+
+
+
+    def exec_command(command)
+      exec_command = "sudo -i -u #{@user} cnvrg exec_container #{@container} \"#{command}\" "
+      return @ssh.exec!(exec_command)
+    end
+
+
+    def close_ssh()
+
+
+      begin
+
+        @ssh.close
+      rescue => e
+
+      end
+
+
+    end
+  end
+
+
+
+end

data/lib/cnvrg/version.rb

@@ -1,5 +1,5 @@
 module Cnvrg
-    VERSION = '0.0.142561'
+    VERSION = '0.0.142562'
 
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.0.142561
+  version: 0.0.142562
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-09 00:00:00.000000000 Z
+date: 2017-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -329,6 +329,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A CLI tool for interacting with cnvrg.io.
 email:
 - info@cnvrg.io
@@ -353,6 +367,7 @@ files:
 - lib/cnvrg/job.rb
 - lib/cnvrg/project.rb
 - lib/cnvrg/runner.rb
+- lib/cnvrg/ssh.rb
 - lib/cnvrg/version.rb
 homepage: https://cnvrg.io
 licenses: []