cnvrg 0.0.1520000 → 0.0.1530000

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8350e6b16c815de7c51ea0020bec8e8ee3f760f4
-  data.tar.gz: 45a2c3cec4e0fa3450b6674a8f554c210db57af7
+  metadata.gz: dfd7e3915a7ca0c235ac844cbb19de4ecd985ff8
+  data.tar.gz: e5435e7a078993757e7e0bcc937632cd73d16ffd
 SHA512:
-  metadata.gz: cb635cb17695ed08f6ee86a175ff18ed5e43fcad39a21f0b3037566605e212c3ff7ec32f535e128e58ba51d9ac446c69d90a9c57450bdede057ebfb48b02aff3
-  data.tar.gz: 20e44b71140893d5aeca636e9f985b92f66471596d9d83a44196f263ffbed2caae9532ffa645aeb5f20158140cadc771bf659201dc91fa9766aa06f7f5eddd3d
+  metadata.gz: 05388102f35ca1434e624cdab9bdad58c52e28594fcc9b5a18edb9ff7a2953341fbcf9e141a1bd751fcfb9136763afba99a0f3adfe841600c84375d674dd7580
+  data.tar.gz: 9350e72abedd72550692d0b4fa65f9f6c95ad5f56d9f343480ce3c360a02b8ee369ba9fc9dac37dda8837f1c581893c3d2755bc37a46c3ddbf0dae23e4cad349

data/lib/cnvrg/Images.rb

@@ -2,6 +2,7 @@ require 'fileutils'
 require 'cnvrg/files'
 require 'docker'
 require 'net/ssh'
+require 'mimemagic'
 
 
 module Cnvrg
@@ -95,18 +96,46 @@ module Cnvrg
 
 
     end
-    def self.create_new_custom_image(type,owner,image_name,is_public,is_base,image_extend,python3)
-      response = Cnvrg::API.request("users/#{owner}/images/custom", 'POST', {instance_type:type,image_name:image_name,is_public:is_public,
-                                                                             is_base:is_base,image_extend:image_extend,
-                                                                             python3:python3})
+    def self.create_new_custom_image(type,owner,image_name,is_public,is_base,image_extend,python3,tar_path)
+       response = Cnvrg::API.request("users/#{owner}/images/custom", 'POST', {instance_type:type,image_name:image_name,is_public:is_public,
+                                                                                is_base:is_base,image_extend:image_extend,
+                                                                                python3:python3})
+
       return response
     end
+    def self.create_new_custom_image_with_docker(type,owner,image_name,is_public,is_base,image_extend,python3,tar_path,files)
+        file_name = File.basename tar_path
+        file_size = File.size(tar_path).to_f
+        mime_type = MimeMagic.by_path(tar_path)
+        content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
+
+        upload_resp = Cnvrg::API.request("/users/#{owner}/images/upload_docker", 'POST_FILE', {absolute_path: tar_path, relative_path: tar_path,
+                                                                                        file_name: file_name, file_size: file_size,
+                                                                                        file_content_type: content_type,
+                                                                                        image_name:image_name,is_public:is_public,
+                                                                                        is_base:is_base,image_extend:image_extend,
+                                                                                        python3:python3 })
+
+        if Cnvrg::CLI.is_response_success(upload_resp, false)
+          path = upload_resp["result"]["path"]
+
+          s3_res = files.upload_small_files_s3(path, tar_path, content_type)
+          if s3_res
+            image_slug = upload_resp["result"]["id"]
+            response = Cnvrg::API.request("users/#{owner}/images/#{image_slug}/build", 'POST', {instance_type:type,image_extend:image_extend})
+          end
+        end
+
+    end
+
+
+
     def self.revoke_custom_new_image(owner,slug)
       response = Cnvrg::API.request("users/#{owner}/images/#{slug}/revoke_image", 'GET')
       return response
     end
-    def self.commit_custom_image(owner,slug)
-      response = Cnvrg::API.request("users/#{owner}/images/#{slug}/commit_custom_image", 'GET')
+    def self.commit_custom_image(owner,slug,logs)
+      response = Cnvrg::API.request("users/#{owner}/images/#{slug}/commit_custom_image", 'POST', {image_logs:logs})
       return response
     end
     def self.ssh_to_machine(resp)

data/lib/cnvrg/cli.rb

@@ -783,7 +783,6 @@ module Cnvrg
           working_dir = "#{Dir.pwd}/#{dataset_name}"
           @dataset = Dataset.new(working_dir)
           @dataset.generate_idx()
-          say "Downloading data", Thor::Shell::Color::BLUE
 
 
           download_data(false, false, path = working_dir, in_dir=false)
@@ -899,6 +898,8 @@ module Cnvrg
           log_end(0)
           return true
         end
+        say "Downloading data", Thor::Shell::Color::BLUE
+
         result = @dataset.downlowd_updated_data(@dataset.last_local_commit)
 
         delete = result["result"]["delete"]
@@ -1711,10 +1712,12 @@ module Cnvrg
                 update_count += 1
                 successful_updates<< relative_path
               else
+                say "Failed to upload: #{ File.basename(absolute_path) }", Thor::Shell::Color::RED
+
                 @files.rollback_commit(commit_sha1)
-                log_end(1, "can't upload, Rolling Back all changes")
+                log_end(1, "Failed to upload: #{ File.basename(absolute_path) }, rolling back")
                 say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
-                exit(0)
+                exit(1)
               end
             end
           end
@@ -1807,7 +1810,7 @@ module Cnvrg
           @files.rollback_commit(commit_sha1)
         end
       rescue => e
-        log_end(-1)
+        log_end(-1,e.message)
 
         say "Error occurd, \nAborting", Thor::Shell::Color::RED
         @files.rollback_commit(commit_sha1)
@@ -2978,7 +2981,6 @@ module Cnvrg
     def remote_notebook()
       verify_logged_in(true)
       log_start(__method__, args, options)
-      verify_software_installed("docker")
 
       working_dir = is_cnvrg_dir()
       notebook_dir = options["notebook_dir"]
@@ -3077,7 +3079,6 @@ module Cnvrg
       begin
         verify_logged_in(true)
         log_start(__method__, args, options)
-        verify_software_installed("docker")
         project_dir = is_cnvrg_dir()
 
         image = is_project_with_docker(project_dir)
@@ -3141,7 +3142,6 @@ module Cnvrg
       begin
         verify_logged_in(true)
         log_start(__method__, args, options)
-        verify_software_installed("docker")
         system = options["system"] || false
 
 
@@ -3206,7 +3206,6 @@ module Cnvrg
       begin
         verify_logged_in(true)
         log_start(__method__, args, options)
-        verify_software_installed("docker")
 
         project_dir = is_cnvrg_dir()
 
@@ -3377,7 +3376,6 @@ module Cnvrg
       begin
         verify_logged_in(true)
         log_start(__method__, args, options)
-        verify_software_installed("docker")
         remote = options["remote"] || false
         project_dir = is_cnvrg_dir()
 
@@ -3449,7 +3447,6 @@ module Cnvrg
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
-        verify_software_installed("docker")
         image = is_project_with_docker(Dir.pwd)
         if image and image.is_docker
           container = image.get_container
@@ -3501,7 +3498,6 @@ module Cnvrg
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
-        verify_software_installed("docker")
         image = is_project_with_docker(Dir.pwd)
         if image and image.is_docker
           container = image.get_container
@@ -3566,10 +3562,12 @@ module Cnvrg
     method_option :public, :type => :boolean, :aliases => ["-p","--public"], :default => false
     method_option :base, :type => :boolean, :aliases => ["-b","--base"], :default => false
     method_option :python3, :type => :boolean, :aliases => ["--python3"], :default => false
+    method_option :docker_path, :type => :string, :aliases => ["--docker_path"], :default => ""
+
 
     desc 'create_custom_image', 'run commands inside containers', :hide=>true
 
-    def create_custom_image(image_name)
+    def build_image(image_name)
       begin
       verify_logged_in(false)
       log_start(__method__, args, options)
@@ -3580,12 +3578,30 @@ module Cnvrg
       public = options["public"]
       base = options["base"]
       python3 = options["python3"]
+      docker_path = options["docker_path"]
       owner = CLI.get_owner
       checks = Helpers.checkmark()
+      tar_path = nil
+      if !docker_path.nil? and !docker_path.empty?
+        docker_path  = File.absolute_path(docker_path)
+        #create tar of the docker path: it could be a docker file, and it could be a docker folder
+        tar_path = File.expand_path('~')+"/.cnvrg/tmp/docker_#{File.basename docker_path}.tar.gz"
+        resp = create_docker_tar(docker_path,tar_path)
+        if !resp
+          say "Couldn't create tar from docker path",  Thor::Shell::Color::RED
+          FileUtils.rm_rf tar_path
+          exit(1)
+        end
+        files = Cnvrg::Files.new(owner, "")
+        resp = Images.create_new_custom_image_with_docker(instance_type,owner,image_name,public,base,image_extend,python3,tar_path,files)
+        if resp
+        end
+      else
+        say "Creating machine for your custom image, this may take a few moments...", Thor::Shell::Color::BLUE
+        resp = Images.create_new_custom_image(instance_type,owner,image_name,public,base,image_extend,python3,nil)
 
-      say "Creating machine for custom image, this may take a few moments...", Thor::Shell::Color::BLUE
+      end
 
-      resp = Images.create_new_custom_image(instance_type,owner,image_name,public,base,image_extend,python3)
       if Cnvrg::CLI.is_response_success(resp,false)
         image_slug = resp["result"]["slug"]
         container = resp["result"]["machine_c"]
@@ -3597,21 +3613,52 @@ module Cnvrg
           Images.revoke_custom_new_image(owner, image_slug)
           log_end(-1,"Couldn't connect to machine,aborting")
         end
-        say "run command until ctrl + c initiates", Thor::Shell::Color::BLUE
+        say "run command until ctrl + c or quit is initiated", Thor::Shell::Color::BLUE
         begin
+          logs = []
 
           while true
             command = ask("$>")
-            puts ssh.exec_command(command)
+            logs <<  { time: Time.now,
+                                 message: command,
+                                 type: "stdout"
+            }
+            if command.eql? "quit"
+              say "Commiting Image..", Thor::Shell::Color::BLUE
+              break
+            end
+            res = ssh.exec_command(command)
+            begin
+              res_parsed = JSON.parse(res)
+              res = res_parsed.join(",")
+            end
+
+            puts res
+            logs <<  { time: Time.now,
+                       message: res,
+                       type: "stdout"
+            }
+            logs.flatten!
+
           end
 
         rescue SignalException
           say "Commiting Image..", Thor::Shell::Color::BLUE
 
         end
-        resp = Images.commit_custom_image(owner,image_slug)
+        resp = Images.commit_custom_image(owner,image_slug,logs)
         if Cnvrg::CLI.is_response_success(resp,false)
           say "#{checks} Image commited successfuly, email will be sent when image is ready", Thor::Shell::Color::GREEN
+        else
+          if image_slug
+            Images.revoke_custom_new_image(owner, image_slug)
+          end
+          if ssh
+            ssh.close_ssh()
+          end
+          say "Image couldn't be commited, rolling back changes", Thor::Shell::Color::RED
+
+          exit(1)
         end
         if ssh
           ssh.close_ssh()
@@ -3655,7 +3702,6 @@ module Cnvrg
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
-        verify_software_installed("docker")
         working_dir = is_cnvrg_dir
         install_file = options["install"] || nil
         if !install_file.nil?
@@ -3730,7 +3776,6 @@ module Cnvrg
     def commit_image
       verify_logged_in(true)
       log_start(__method__, args, options)
-      verify_software_installed("docker")
 
       begin
         image = is_project_with_docker(Dir.pwd)
@@ -3776,7 +3821,6 @@ module Cnvrg
     def sync_image(docker=false)
       verify_logged_in(true)
       log_start(__method__, args, options)
-      verify_software_installed("docker")
       is_public = options["is_public"] || false
       is_base = options["is_base"] || false
       message = options["message"] || ""
@@ -3848,7 +3892,6 @@ module Cnvrg
     def upload_image(image_id, is_public, is_base, *message)
       verify_logged_in(true)
       log_start(__method__, args, options)
-      verify_software_installed("docker")
       image = Docker::Image.get(image_id)
       project_home = get_project_home
       @project = Project.new(project_home)
@@ -4197,7 +4240,6 @@ module Cnvrg
     def upload_cnvrg_image(image_name)
       verify_logged_in(false)
       log_start(__method__, args, options)
-      verify_software_installed("docker")
       owner = Cnvrg::CLI.get_owner()
 
       path = File.expand_path('~')+"/.cnvrg/tmp/#{image_name}.zip"
@@ -4648,8 +4690,11 @@ module Cnvrg
         if response.nil?
           if !Cnvrg::Helpers.internet_connection?
             say("<%= color('Error:You seems to be offline', RED) %>")
+          end
+          if should_exit
+            exit(1)
           else
-            say("<%= color('Error', RED) %>")
+            return false
           end
         elsif response["status"]!= 200
           error = response['message']
@@ -4746,10 +4791,11 @@ module Cnvrg
               logfile_old = File.expand_path('~') +"/.cnvrg/log_#{date}.log"
               count+=1
             end
-            @files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
-            @files.upload_log_file(logfile_old, "log_#{date}.log", yesterday)
-            FileUtils.remove logfile_old
-
+            if File.exist? logfile_old
+              @files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
+              @files.upload_log_file(logfile_old, "log_#{date}.log", yesterday)
+              FileUtils.remove logfile_old
+            end
 
           end
           $LOG = LogStashLogger.new(type: :file, path: logfile, sync: true)
@@ -5063,6 +5109,21 @@ module Cnvrg
           return relative
         end
       end
+      def create_docker_tar(docker_path,tar_path)
+        begin
+
+        if File.directory? (docker_path)
+          `cd #{docker_path} && tar -czf #{tar_path} . `
+        else
+          dir_name = File.dirname docker_path
+          `cd #{dir_name} &&  tar -czf #{tar_path} #{File.basename(docker_path)}`
+        end
+        rescue => e
+          puts "Exception while compressing docker path: #{e.message}"
+        end
+
+        return $?.success?
+      end
 
       def create_tar(path_in, path_out, tar_files,no_compression=false)
         #The cd is meant for cases when running cnvrg data uplaod not in the main folder

data/lib/cnvrg/datafiles.rb

@@ -64,13 +64,14 @@ module Cnvrg
 
         upload_resp = Cnvrg::API.request(@base_resource + "upload_tar_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
                                                                                            commit_sha1: commit_sha1, file_name: file_name,
-                                                                                           file_size: file_size, file_content_type: content_type, sha1: sha1})
+                                                                                           file_size: file_size, file_content_type: content_type, sha1: sha1,
+                                                                                            new_version:true})
         if Cnvrg::CLI.is_response_success(upload_resp, false)
           path = upload_resp["result"]["path"]
           s3_res = upload_large_files_s3(upload_resp, absolute_path)
           if s3_res
-            Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
-                                                                      blob_id: upload_resp["result"]["id"]})
+            # Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
+            #                                                           blob_id: upload_resp["result"]["id"]})
             return true
           end
         else
@@ -283,16 +284,29 @@ module Cnvrg
         home_dir = File.expand_path('~')
         log_file = "#{home_dir}/.cnvrg/tmp/upload_#{File.basename(file_path)}.log"
 
-        URLcrypt::key = [body].pack('H*')
-        client = Aws::S3::Client.new(:access_key_id => URLcrypt.decrypt(upload_resp["result"]["new_a"]),
-                                     :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["new_s"]),
-                                     :region => URLcrypt.decrypt(upload_resp["result"]["region"]),
+
+        split = body.split("\n")
+        key = split[0]
+        iv = split[1]
+
+        access =  Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["new_a"])
+
+        secret =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["new_s"])
+
+        region =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["region"])
+
+        bucket =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["bucket"])
+
+
+        client = Aws::S3::Client.new(:access_key_id => access,
+                                     :secret_access_key =>secret,
+                                     :region => region,
                                      :logger => Logger.new(log_file),
                                      :http_open_timeout => 60, :retry_limit => 20,
                                      :http_wire_trace => true)
         s3 = Aws::S3::Resource.new(client: client)
 
-        resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
+        resp = s3.bucket(bucket).
             object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
             upload_file(file_path, {:use_accelerate_endpoint => true})
         # s4cmd_path = upload_resp["result"]["path_s4cmd"]
@@ -407,15 +421,33 @@ module Cnvrg
             response = http.request request
             body = response.read_body
           end
-          URLcrypt::key = [body].pack('H*')
-          s3 = Aws::S3::Client.new(
-              :access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
-              :secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
-              :session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
-              :region => URLcrypt.decrypt(download_resp["result"]["region"]))
+          split = body.split("\n")
+          key = split[0]
+          iv = split[1]
+
+          access =  Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
+
+          secret =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_s"])
+
+          session =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_st"])
+          region =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
+
+          bucket =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
+          key =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
+
+
+
+          client = Aws::S3::Client.new(
+              :access_key_id =>access,
+              :secret_access_key => secret,
+              :session_token => session,
+              :region => region,
+              :http_open_timeout => 60, :retry_limit => 20
+          )
+
           File.open(project_home+"/"+absolute_path, 'wb') do |file|
-            resp = s3.get_object({bucket: URLcrypt.decrypt(download_resp["result"]["bucket"]),
-                                  key: URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
+            resp = client.get_object({bucket:bucket,
+                                  key:key}, target: file)
           end
           return true
         end
@@ -428,7 +460,7 @@ module Cnvrg
 
     def download_data_file(commit_sha1, dataset_home)
       begin
-        res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1: commit_sha1})
+        res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1: commit_sha1,new_version:true})
         Cnvrg::CLI.is_response_success(res, false)
         if res["result"]
           download_resp = res
@@ -445,16 +477,34 @@ module Cnvrg
             response = http.request request
             body = response.read_body
           end
-          URLcrypt::key = [body].pack('H*')
-          s3 = Aws::S3::Client.new(
-              :access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
-              :secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
-              :session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
-              :region => URLcrypt.decrypt(download_resp["result"]["region"]))
+          split = body.split("\n")
+          key = split[0]
+          iv = split[1]
+
+          access =  Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
+
+          secret =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_s"])
+
+          session =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_st"])
+          region =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
+
+          bucket =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
+          key =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
+
+
+
+          client = Aws::S3::Client.new(
+              :access_key_id =>access,
+              :secret_access_key => secret,
+              :session_token => session,
+              :region => region,
+              :http_open_timeout => 60, :retry_limit => 20
+          )
+
 
           File.open(dataset_home+"/"+filename, 'wb') do |file|
-            resp = s3.get_object({bucket: URLcrypt.decrypt(download_resp["result"]["bucket"]),
-                                  key: URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
+            resp = client.get_object({bucket: bucket,
+                                  key: key }, target: file)
           end
           return filename
         end

data/lib/cnvrg/files.rb

@@ -26,7 +26,8 @@ module Cnvrg
 
       upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
                                                                                      commit_sha1: commit_sha1, file_name: file_name,
-                                                                                     file_size: file_size, file_content_type: content_type, sha1: sha1})
+                                                                                     file_size: file_size, file_content_type: content_type, sha1: sha1,
+                                                                                      new_version:true})
       if Cnvrg::CLI.is_response_success(upload_resp, false)
         path = upload_resp["result"]["path"]
         if file_size.to_f>= Cnvrg::Files::LARGE_FILE.to_f
@@ -241,7 +242,6 @@ module Cnvrg
 
 
           sts_path = upload_resp["result"]["path_sts"]
-          s4cmd_path = upload_resp["result"]["path_s4cmd"]
 
           uri = URI.parse(sts_path)
           http_object = Net::HTTP.new(uri.host, uri.port)
@@ -253,77 +253,45 @@ module Cnvrg
             response = http.request request
             body = response.read_body
           end
+          split = body.split("\n")
+          key = split[0]
+          iv = split[1]
 
-          URLcrypt::key = [body].pack('H*')
-          is_python = false
-          s4cmd_suc = false
-          s4cmd_install_suc = false
-          # python_version=`python --version  > /dev/null 2>&1`; is_python=$?.success?
-          # if is_python
-          #
-          #   s4cmd=`pip freeze 2>/dev/null |grep -e s4cmd -e boto3 > /dev/null 2>&1`; s4cmd_suc=$?.success?
-          #   if !s4cmd_suc
-          #     `pip install s4cmd  > /dev/null 2>&1`; s4cmd_install_suc=$?.success?
-          #   end
-          #
-          # end
-          if !s4cmd_suc and !s4cmd_install_suc
-            s3 = Aws::S3::Resource.new(
-                :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
-                :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
-                :session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
-                :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
-            resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
-                object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
-                upload_file(file_path, {:use_accelerate_endpoint => true})
-          else
-            s4cmd_uri = URI.parse(s4cmd_path)
-            s4cmd_http_object = Net::HTTP.new(s4cmd_uri.host, s4cmd_uri.port)
-            s4cmd_http_object.use_ssl = true if s4cmd_uri.scheme == 'https'
-            s4cmd_request = Net::HTTP::Get.new(s4cmd_path)
-
-            s4cmd_body = ""
-            s4cmd_http_object.start do |http|
-              response = http.request s4cmd_request
-              s4cmd_body = response.read_body
-            end
-            s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
-                                      aws_access_key_id=aws_access_key_id,
-                                      aws_secret_access_key=aws_secret_access_key)", " self.client = self.boto3.client('s3',
-                                      aws_access_key_id='#{ URLcrypt.decrypt(upload_resp["result"]["sts_a"])}',
-                                      aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["sts_s"])}',
-                                      aws_session_token='#{URLcrypt.decrypt(upload_resp["result"]["sts_st"])}')")
-
-
-            tmp = Tempfile.new('s4cmd.py')
-            tmp << s4cmd_new_body
-            tmp.flush
-            tmp.close
-            #
-            is_success = false
-            count = 0
-            while !is_success and count <3
-              resp = `python #{tmp.path} --num-threads=8 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)} > /dev/null 2>&1 `
-              is_success =$?.success?
-              count +=1
+          access =  Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_a"])
 
-            end
-            resp= is_success
+          secret =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["sts_s"])
+
+          session =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["sts_st"])
+          region =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["region"])
+
+          bucket =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["bucket"])
 
-          end
+
+
+          client = Aws::S3::Client.new(
+                :access_key_id =>access,
+                :secret_access_key => secret,
+                :session_token => session,
+                :region => region,
+                :http_open_timeout => 60, :retry_limit => 20
+                                         )
+            s3 = Aws::S3::Resource.new(client: client)
+            resp = s3.bucket(bucket).
+                object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
+                upload_file(file_path, {:use_accelerate_endpoint => true})
 
           return resp
 
-          rescue => e
-          if File.exist? tmp
-            FileUtils.rm_rf [tmp]
-          end
-          return false
+      rescue => e
+        puts e
+        puts e.backtrace
+        return false
 
         end
         return true
 
-      end
+    end
+
 
       def upload_small_files_s3(url_path, file_path, content_type)
         url = URI.parse(url_path)
@@ -370,7 +338,8 @@ module Cnvrg
 
       def download_file_s3(absolute_path, relative_path, project_home, commit_sha1=nil, conflict=false)
         begin
-          res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
+          res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path,
+                                                                              commit_sha1: commit_sha1,new_version:true})
           Cnvrg::CLI.is_response_success(res, false)
           if res["result"]
             download_resp = res
@@ -388,20 +357,39 @@ module Cnvrg
               response = http.request request
               body = response.read_body
             end
-            URLcrypt::key = [body].pack('H*')
-            s3 = Aws::S3::Client.new(
-                :access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
-                :secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
-                :session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
-                :region => URLcrypt.decrypt(download_resp["result"]["region"]))
+            split = body.split("\n")
+            key = split[0]
+            iv = split[1]
+
+            access =  Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
+
+            secret =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_s"])
+
+            session =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_st"])
+            region =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
+
+            bucket =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
+            key =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
+
+
+
+            client = Aws::S3::Client.new(
+                :access_key_id =>access,
+                :secret_access_key => secret,
+                :session_token => session,
+                :region => region,
+                :http_open_timeout => 60, :retry_limit => 20
+            )
+
             File.open(project_home+"/"+absolute_path, 'wb') do |file|
-              resp = s3.get_object({bucket: URLcrypt.decrypt(download_resp["result"]["bucket"]),
-                                    key: URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
+              resp = client.get_object({bucket:bucket,
+                                    key:key}, target: file)
             end
             return true
           end
 
         rescue => e
+          puts e
           return false
 
         end

data/lib/cnvrg/helpers.rb

@@ -171,6 +171,19 @@ parameters:
       Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond)
     end
 
+    def decrypt(key,iv,str)
+      cipher = OpenSSL::Cipher.new("aes-256-cbc").decrypt
+      cipher.key = key
+      cipher.iv = Base64.decode64 iv.encode('ascii-8bit')
+
+      result = Base64.decode64 (str.encode('ascii-8bit'))
+      result = cipher.update(result)
+      result << cipher.final
+
+      return result
+
+    end
+
     # memory
     #
     def get_mem(pid)

data/lib/cnvrg/version.rb

@@ -1,5 +1,5 @@
 module Cnvrg
-    VERSION = '0.0.1520000'
+    VERSION = '0.0.1530000'
 
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.0.1520000
+  version: 0.0.1530000
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-23 00:00:00.000000000 Z
+date: 2017-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler