cnvrg 0.0.15000 → 0.0.142561

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 209fd646618556d6e2a6ed54778d95ba9d9fb5ae
-  data.tar.gz: 8c7df0eea5f08eaec0a8d50326f20d3fbcfbb47d
+  metadata.gz: d2a92149e9b565661704f3f91438b28c89976475
+  data.tar.gz: 9f5c2dc4a71823c9b5657fd941361e33305d06f6
 SHA512:
-  metadata.gz: 77dd8ba073bcc0ec65d6dc4a70845ae32a3a779f8b1b340dcbaeee4336676db5d1a928b38a23eb619df396c413a613c69378f9deb114e7567e8ebab58607c79d
-  data.tar.gz: c876d3596e5287e6d28d6f972f777188b1f2b92c8001aaa178da2db31a3fb254edf8827addb128b4e4ab4dd5c5eeafed481362e2c02af3e0081e7f4278553b09
+  metadata.gz: 0aace663fb6460ade819c6676fb96b11cbbb286340e0b2599b6be6bb6d1ae5f3e6098270b9f37976cae7ec743b6381f46dd1e1054cb4f56e71d249cbb875f43b
+  data.tar.gz: 47dd9900f83b208196690862ceba2511643fbf84fc2d1daa6a23b66c51c63279aa9766bcaee9811b72af9b4f44e605e7fa34e5b0e450bcbe67f4e62a7480c173

data/cnvrg.gemspec

@@ -40,8 +40,6 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'rubyzip', '~> 1.2'
     spec.add_runtime_dependency 'activesupport', '~> 5.0'
     spec.add_runtime_dependency 'ruby-progressbar'
-    spec.add_runtime_dependency 'net-ssh'
-
 
 
     

data/lib/cnvrg.rb

@@ -3,5 +3,6 @@ require 'cnvrg/version'
 require 'cnvrg/cli'
 require 'thor'
 
+
 module Cnvrg
 end

data/lib/cnvrg/Images.rb

@@ -1,8 +1,6 @@
 require 'fileutils'
 require 'cnvrg/files'
 require 'docker'
-require 'net/ssh'
-
 
 module Cnvrg
   class Images
@@ -95,73 +93,6 @@ module Cnvrg
 
 
     end
-    def self.create_new_custom_image(type,owner,image_name,is_public,is_base,image_extend,python3)
-      response = Cnvrg::API.request("users/#{owner}/images/custom", 'POST', {instance_type:type,image_name:image_name,is_public:is_public,
-                                                                             is_base:is_base,image_extend:image_extend,
-                                                                             python3:python3})
-      return response
-    end
-    def self.revoke_custom_new_image(owner,slug)
-      response = Cnvrg::API.request("users/#{owner}/images/#{slug}/revoke_image", 'GET')
-      return response
-    end
-    def self.commit_custom_image(owner,slug)
-      response = Cnvrg::API.request("users/#{owner}/images/#{slug}/commit_custom_image", 'GET')
-      return response
-    end
-    def self.ssh_to_machine(resp)
-
-      sts_path = resp["result"]["sts_path"]
-
-      uri = URI.parse(sts_path)
-
-      http_object = Net::HTTP.new(uri.host, uri.port)
-      http_object.use_ssl = true if uri.scheme == 'https'
-      request = Net::HTTP::Get.new(sts_path)
-
-      body = ""
-      http_object.start do |http|
-        response = http.request request
-        body = response.read_body
-      end
-
-      URLcrypt::key = [body].pack('H*')
-
-      ip = URLcrypt.decrypt(resp["result"]["machine_i"])
-
-      user = URLcrypt.decrypt(resp["result"]["machine_u"])
-      key = URLcrypt.decrypt(resp["result"]["machine_k"])
-      tempssh = Tempfile.new "sshkey"
-      tempssh.write open(key).read
-      tempssh.rewind
-      key_path = tempssh.path
-      count = 0
-      while count < 5
-
-      begin
-          ssh = Net::SSH.start(ip, user=user, :keys => key_path, :timeout => 10)
-          if !ssh.nil?
-            return ssh
-          else
-            count+=1
-            sleep(2)
-
-          end
-        rescue
-      count+=1
-      sleep(2)
-
-
-      end
-      end
-      if tempssh
-        tempssh.close
-        tempssh.unlink
-      end
-      return false
-    end
-
-
 
     def create_custom_image(new_image_name,working_dir,stored_commands)
 

data/lib/cnvrg/api.rb

@@ -57,10 +57,7 @@ module Cnvrg
                 else
                     response
                 end
-              when 'POST'
-                conn.options.timeout = 420
-                conn.options.open_timeout =420
-
+            when 'POST'
                 response = conn.post "#{endpoint_uri}/#{resource}", data
 
                 if parse_request == true
@@ -73,12 +70,12 @@ module Cnvrg
                     fr.headers['Auth-Token'] = @pass
                     fr.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
                     fr.headers["Content-Type"] = "multipart/form-data"
-
+                    
                     fr.request :multipart
                     fr.request :url_encoded
                     fr.adapter :net_http
                 end
-
+                
 
                 # what if windows?
                 # data[:file] = Faraday::UploadIO.new(data[:absolute_path], content_type)
@@ -107,7 +104,6 @@ module Cnvrg
             else
             end
             rescue => e
-              puts e
                return nil
             end
 

data/lib/cnvrg/cli.rb

@@ -21,7 +21,6 @@ require 'cnvrg/Images'
 require 'cnvrg/dataset'
 require 'cnvrg/datafiles'
 require 'cnvrg/data'
-require 'cnvrg/ssh'
 require 'etc'
 require 'logstash-logger'
 require 'cnvrg/job'
@@ -1155,9 +1154,8 @@ module Cnvrg
     method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
-    method_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
 
-    def upload_data_tar(ignore, verbose, sync,no_compression)
+    def upload_data_tar(ignore, verbose, sync)
 
       begin
         verify_logged_in(true)
@@ -1227,7 +1225,7 @@ module Cnvrg
           tar_files_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.txt"
           tar_files = (result["added"] + result["updated_on_local"]).join("\n")
           File.open(tar_files_path, 'w') { |f| f.write tar_files }
-          is_tar = create_tar(dataset_dir, tar_path, tar_files_path,no_compression)
+          is_tar = create_tar(dataset_dir, tar_path, tar_files_path)
           if !is_tar
             say "ERROR: Couldn't compress data", Thor::Shell::Color::RED
             FileUtils.rm_rf([tar_path]) if File.exist? tar_path
@@ -3543,6 +3541,7 @@ module Cnvrg
           container.stop()
         end
       rescue SignalException
+        log_End(-1)
         if container
           container.stop()
         end
@@ -3552,98 +3551,6 @@ module Cnvrg
 
     end
 
-    method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
-    method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
-    method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
-    method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
-    method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
-    method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
-    method_option :image, :type => :string, :aliases => ["-i","--image"], :default => ""
-    method_option :public, :type => :boolean, :aliases => ["-p","--public"], :default => false
-    method_option :base, :type => :boolean, :aliases => ["-b","--base"], :default => false
-    method_option :python3, :type => :boolean, :aliases => ["--python3"], :default => false
-
-    desc 'create_custom_image', 'run commands inside containers', :hide=>true
-
-    def create_custom_image(image_name)
-      begin
-      verify_logged_in(false)
-      log_start(__method__, args, options)
-      instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
-                   "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
-      instance_type = get_instance_type(instances)
-      image_extend = options["image"]
-      public = options["public"]
-      base = options["base"]
-      python3 = options["python3"]
-      owner = CLI.get_owner
-      checks = Helpers.checkmark()
-
-      say "Creating machine for custom image, this may take a few moments...", Thor::Shell::Color::BLUE
-
-      resp = Images.create_new_custom_image(instance_type,owner,image_name,public,base,image_extend,python3)
-      if Cnvrg::CLI.is_response_success(resp,false)
-        image_slug = resp["result"]["slug"]
-        container = resp["result"]["machine_c"]
-        say "#{checks} Created image and machine successfully", Thor::Shell::Color::GREEN
-        say "Connecting to machine", Thor::Shell::Color::BLUE
-        ssh = Ssh.new(resp)
-        if !ssh.is_ssh
-          say "Couldn't connect to machine,aborting", Thor::Shell::Color::RED
-          Images.revoke_custom_new_image(owner, image_slug)
-          log_end(-1,"Couldn't connect to machine,aborting")
-        end
-        say "run command until ctrl + c initiates", Thor::Shell::Color::BLUE
-        begin
-
-          while true
-            command = ask("$>")
-            puts ssh.exec_command(command)
-          end
-
-        rescue SignalException
-          say "Commiting Image..", Thor::Shell::Color::BLUE
-
-        end
-        resp = Images.commit_custom_image(owner,image_slug)
-        if Cnvrg::CLI.is_response_success(resp,false)
-          say "#{checks} Image commited successfuly, email will be sent when image is ready", Thor::Shell::Color::GREEN
-        end
-        if ssh
-          ssh.close_ssh()
-        end
-
-
-
-
-
-
-
-      end
-      rescue =>e
-        puts e
-        if image_slug
-          Images.revoke_custom_new_image(owner, image_slug)
-        end
-        if ssh
-          ssh.close_ssh()
-        end
-
-
-      rescue SignalException
-        if image_slug
-          Images.revoke_custom_new_image(owner, image_slug)
-        end
-        if ssh
-          ssh.close_ssh
-        end
-        say "\nAborting"
-        exit(1)
-      end
-
-    end
-
-
     desc 'build', 'run commands inside containers'
     method_option :install, :type => :string, :aliases => ["--i"], :default => nil, :desc => "Install from the given instructions file"
 
@@ -5045,14 +4952,10 @@ module Cnvrg
         end
       end
 
-      def create_tar(path_in, path_out, tar_files,no_compression=false)
+      def create_tar(path_in, path_out, tar_files)
         #The cd is meant for cases when running cnvrg data uplaod not in the main folder
         begin
-          if no_compression
-            `cd #{path_in} && tar -cf #{path_out} -T #{tar_files}`
-          else
-            `cd #{path_in} && tar -czf #{path_out} -T #{tar_files}`
-          end
+        `cd #{path_in} && tar -czf #{path_out} -T #{tar_files}`
         rescue => e
           puts "Exception while compressing data: #{e.message}"
         end

data/lib/cnvrg/data.rb

@@ -11,9 +11,7 @@ class SubCommandBase < Thor
 end
 
 module Cnvrg
-
   class Data < SubCommandBase
-    class_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
     desc "data init", "init data folder"
     method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
     def init
@@ -25,15 +23,13 @@ module Cnvrg
     method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
-    method_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
-
     def upload
       cli = Cnvrg::CLI.new()
       ignore = options["ignore"]
       verbose = options["verbose"]
       sync = options["sync"]
-      no_compression = options["no_compression"]
-      cli.upload_data_tar(ignore, verbose,sync,no_compression)
+
+      cli.upload_data_tar(ignore, verbose,sync)
     end
     desc 'data download', 'pull data'
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false

data/lib/cnvrg/datafiles.rb

@@ -234,7 +234,7 @@ module Cnvrg
     def   upload_large_files_s3(upload_resp, file_path)
       begin
         sts_path = upload_resp["result"]["path_sts"]
-        # s4cmd_path = upload_resp["result"]["path_s4cmd"]
+        s4cmd_path = upload_resp["result"]["path_s4cmd"]
 
         uri = URI.parse(sts_path)
         http_object = Net::HTTP.new(uri.host, uri.port)
@@ -261,51 +261,51 @@ module Cnvrg
         #   end
         #
         # end
-        resp = false
 
         if  !s4cmd_suc and !s4cmd_install_suc
           s3 = Aws::S3::Resource.new(
-              :access_key_id => URLcrypt.decrypt(upload_resp["result"]["new_a"]),
-              :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["new_s"]),
+              :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
+              :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
+              :session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
               :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
           resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
               object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
               upload_file(file_path,{:use_accelerate_endpoint=>true})
 
-        # else
-
-          # s4cmd_uri = URI.parse(s4cmd_path)
-          # s4cmd_http_object = Net::HTTP.new(s4cmd_uri.host, s4cmd_uri.port)
-          # s4cmd_http_object.use_ssl = true if s4cmd_uri.scheme == 'https'
-          # s4cmd_request = Net::HTTP::Get.new(s4cmd_path)
-          #
-          # s4cmd_body = ""
-          # s4cmd_http_object.start do |http|
-          #   response = http.request s4cmd_request
-          #   s4cmd_body = response.read_body
-          # end
-          #
-          # s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
-          #                             aws_access_key_id=aws_access_key_id,
-          #                             aws_secret_access_key=aws_secret_access_key)"," self.client = self.boto3.client('s3',
-          #                             aws_access_key_id='#{ URLcrypt.decrypt(upload_resp["result"]["sts_a"])}',
-          #                             aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["sts_s"])}',
-          #                             aws_session_token='#{URLcrypt.decrypt(upload_resp["result"]["sts_st"])}')")
-          #
-          # tmp = Tempfile.new('s4cmd.py')
-          # tmp << s4cmd_new_body
-          # tmp.flush
-          # tmp.close
-          #
-          #   is_success = false
-          # count = 0
-          # while !is_success and count <3
-          #   resp = `python #{tmp.path} --num-threads=8 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)}  > /dev/null 2>&1`
-          #   is_success =$?.success?
-          #   count +=1
-          #
-          # end
-          # resp= is_success
+        else
+
+          s4cmd_uri = URI.parse(s4cmd_path)
+          s4cmd_http_object = Net::HTTP.new(s4cmd_uri.host, s4cmd_uri.port)
+          s4cmd_http_object.use_ssl = true if s4cmd_uri.scheme == 'https'
+          s4cmd_request = Net::HTTP::Get.new(s4cmd_path)
+
+          s4cmd_body = ""
+          s4cmd_http_object.start do |http|
+            response = http.request s4cmd_request
+            s4cmd_body = response.read_body
+          end
+
+          s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
+                                      aws_access_key_id=aws_access_key_id,
+                                      aws_secret_access_key=aws_secret_access_key)"," self.client = self.boto3.client('s3',
+                                      aws_access_key_id='#{ URLcrypt.decrypt(upload_resp["result"]["sts_a"])}',
+                                      aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["sts_s"])}',
+                                      aws_session_token='#{URLcrypt.decrypt(upload_resp["result"]["sts_st"])}')")
+
+          tmp = Tempfile.new('s4cmd.py')
+          tmp << s4cmd_new_body
+          tmp.flush
+          tmp.close
+
+            is_success = false
+          count = 0
+          while !is_success and count <3
+            resp = `python #{tmp.path} --num-threads=8 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)}  > /dev/null 2>&1`
+            is_success =$?.success?
+            count +=1
+
+          end
+          resp= is_success
 
         end
 
@@ -313,9 +313,9 @@ module Cnvrg
 
       rescue => e
         puts e.message
-        # if File.exist? tmp
-        #   FileUtils.rm_rf [tmp]
-        # end
+        if File.exist? tmp
+          FileUtils.rm_rf [tmp]
+        end
         return false
 
       end

data/lib/cnvrg/version.rb

@@ -1,5 +1,5 @@
 module Cnvrg
-    VERSION = '0.0.15000'
+    VERSION = '0.0.142561'
 
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.0.15000
+  version: 0.0.142561
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-16 00:00:00.000000000 Z
+date: 2017-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -329,20 +329,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: net-ssh
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: A CLI tool for interacting with cnvrg.io.
 email:
 - info@cnvrg.io
@@ -367,7 +353,6 @@ files:
 - lib/cnvrg/job.rb
 - lib/cnvrg/project.rb
 - lib/cnvrg/runner.rb
-- lib/cnvrg/ssh.rb
 - lib/cnvrg/version.rb
 homepage: https://cnvrg.io
 licenses: []
@@ -388,9 +373,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.
 test_files: []
-has_rdoc: 

data/lib/cnvrg/ssh.rb

@@ -1,95 +0,0 @@
-require 'fileutils'
-require 'cnvrg/files'
-require 'docker'
-require 'net/ssh'
-
-
-module Cnvrg
-  class Ssh
-    attr_reader :is_ssh
-
-
-    def initialize(resp)
-      begin
-        @is_ssh = false
-        sts_path = resp["result"]["sts_path"]
-
-        uri = URI.parse(sts_path)
-
-        http_object = Net::HTTP.new(uri.host, uri.port)
-        http_object.use_ssl = true if uri.scheme == 'https'
-        request = Net::HTTP::Get.new(sts_path)
-
-        body = ""
-        http_object.start do |http|
-          response = http.request request
-          body = response.read_body
-        end
-
-        URLcrypt::key = [body].pack('H*')
-        ip = URLcrypt.decrypt(resp["result"]["machine_i"])
-
-        @user = URLcrypt.decrypt(resp["result"]["machine_u"])
-        key = URLcrypt.decrypt(resp["result"]["machine_k"])
-        @container = URLcrypt.decrypt(resp["result"]["machine_c"])
-
-        tempssh = Tempfile.new "sshkey"
-        tempssh.write open(key).read
-        tempssh.rewind
-        key_path = tempssh.path
-        count = 0
-        while count < 5
-
-          begin
-            @ssh = Net::SSH.start(ip, user=@user, :keys => key_path, :timeout => 10)
-            if !@ssh.nil?
-              @is_ssh = true
-              return
-            else
-              count+=1
-              sleep(2)
-
-            end
-          rescue
-            count+=1
-            sleep(2)
-
-
-          end
-        end
-        if tempssh
-          tempssh.close
-          tempssh.unlink
-        end
-        return false
-      rescue => e
-
-        puts e
-      end
-    end
-
-
-
-    def exec_command(command)
-      exec_command = "sudo -i -u #{@user} cnvrg exec_container #{@container} \"#{command}\" "
-      return @ssh.exec!(exec_command)
-    end
-
-
-    def close_ssh()
-
-
-      begin
-
-        @ssh.close
-      rescue => e
-
-      end
-
-
-    end
-  end
-
-
-
-end