cnvrg 0.0.14 → 0.0.15

data/lib/cnvrg/dataset.rb

@@ -0,0 +1,296 @@
+require 'fileutils'
+module Cnvrg
+  class Dataset
+    attr_reader :slug, :owner, :title, :local_path, :working_dir
+
+    RemoteURL ||= "https://cnvrg.io"
+
+    def initialize(project_home)
+      begin
+        config = YAML.load_file(project_home+"/.cnvrg/config.yml")
+        @local_path = project_home
+        @title = config[:dataset_name]
+        @slug = config[:dataset_slug]
+        @owner = config[:owner]
+        @working_dir = project_home
+      rescue => e
+
+      end
+
+    end
+
+    def last_local_commit
+      idx = YAML.load_file(@local_path + "/.cnvrg/idx.yml")
+      return idx[:commit]
+    end
+
+    def snapshot
+      commit = last_local_commit
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/volumes/create", 'POST', {data_commit: commit})
+      CLI.is_response_success(response)
+      return response
+
+    end
+    def list(owner)
+      response = Cnvrg::API.request("users/#{owner}/datasets/list", 'GET')
+      CLI.is_response_success(response)
+      return response
+
+    end
+    def list_commits
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/list_commits", 'GET')
+      CLI.is_response_success(response)
+      return response
+
+    end
+    def url
+      url = Cnvrg::Helpers.remote_url
+      "#{url}/#{self.owner}/projects/#{self.slug}"
+    end
+
+    def update_ignore_list(new_ignore)
+
+      if new_ignore.nil? or new_ignore.empty?
+        return true
+      end
+      begin
+        File.open(self.local_path+"/.cnvrgignore", "a+") do |f|
+          f.puts("\n")
+
+          new_ignore.each do |i|
+            f.puts("#{i}\n")
+          end
+        end
+        return true
+      rescue
+        return false
+      end
+    end
+
+    def get_ignore_list
+      ignore_list = []
+      File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
+        line = line.strip
+
+        if line.start_with? "#" or ignore_list.include? line
+          next
+        end
+        if line.end_with? "/" or File.directory?(line)
+          if line.end_with? "/"
+            ignore_list << line.chop
+          else
+            ignore_list << line
+          end
+          all_sub = Dir.glob("#{line}/**/*", File::FNM_DOTMATCH).flatten
+          ignore_list << all_sub.flatten
+          ignore_list << line
+        else
+          ignore_list << line
+        end
+      end
+      return ignore_list.flatten
+
+    end
+
+
+    def self.init(owner, dataset_name, is_public=false)
+        list_dirs = [".cnvrg"
+        ]
+        list_files = [
+            ".cnvrgignore",
+            ".cnvrg/config.yml"
+        ]
+
+        cnvrgignore = Helpers.cnvrgignore_content
+        begin
+            response = Cnvrg::API.request("cli/create_dataset", 'POST', {title: dataset_name, owner: owner, is_public: is_public})
+            Cnvrg::CLI.is_response_success(response)
+          response = JSON.parse response["result"]
+          dataset_slug = response["slug"]
+
+          config = {dataset_name: dataset_name,
+                    dataset_slug: dataset_slug,
+                    owner: owner}
+          FileUtils.mkdir_p list_dirs
+          FileUtils.touch list_files
+          File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+          File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
+        rescue => e
+          "Exception in init data: #{e.message}"
+          return false
+        end
+        return true
+    end
+    def self.clone(owner, dataset_name,dataset_slug)
+
+      begin
+        list_dirs = [ dataset_name, "#{dataset_name}/.cnvrg"
+        ]
+        list_files = [
+            "#{dataset_name}/.cnvrgignore",
+            "#{dataset_name}/.cnvrg/config.yml"
+        ]
+
+        config = {dataset_name: dataset_name,
+                  dataset_slug: dataset_slug,
+                  owner: owner}
+
+
+        cnvrgignore = Helpers.cnvrgignore_content
+        FileUtils.mkdir_p list_dirs
+        FileUtils.touch list_files
+        File.open("#{dataset_name}/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+        File.open("#{dataset_name}/.cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
+      rescue => e
+        puts "Exception in clone request:#{e.message}"
+        return false
+      end
+      return true
+    end
+
+    def self.init_container(owner, dataset_slug,dataset_name)
+
+      cnvrgignore = Helpers.cnvrgignore_content
+      begin
+        list_dirs = [ ".cnvrg"
+        ]
+        list_files = [
+            ".cnvrgignore",
+            ".cnvrg/config.yml"
+        ]
+        FileUtils.mkdir_p list_dirs
+        FileUtils.touch list_files
+
+        config = {dataset_name: dataset_name,
+                  dataset_slug: dataset_slug,
+                  owner: owner}
+        File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+
+        File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
+      rescue => e
+        return false
+      end
+      return true
+    end
+
+
+      def get_idx
+        YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+      end
+    def url
+      url = Cnvrg::Helpers.remote_url
+      "#{url}/#{self.owner}/datasets/#{self.slug}"
+    end
+    def generate_idx
+      if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
+        old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+      else
+        old_idx = nil
+      end
+
+      tree_idx = Hash.new(0)
+
+      list = Dir.glob("#{self.local_path}/**/**", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) }
+      list_ignore = self.get_ignore_list()
+      list.each do |e|
+        label = e.gsub(self.local_path + "/", "")
+        if File.directory? e
+          if list_ignore.include? label
+            next
+          end
+          tree_idx[label+"/"] = nil
+        else
+          if list_ignore.include? label
+            next
+          end
+          sha1 = Digest::SHA1.file(e).hexdigest
+          if old_idx.nil? or old_idx.to_h["tree"].nil?
+            tree_idx[label] = {sha1: sha1, commit_time: nil}
+          elsif old_idx["tree"][label].nil? or old_idx["tree"][label]["sha1"] != sha1
+            tree_idx[label] = {sha1: sha1, commit_time: nil}
+          else
+            tree_idx[label] = old_idx["tree"][label]
+          end
+        end
+      end
+
+      idx = {commit: old_idx.to_h[:commit], tree: tree_idx}
+
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
+      return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+    end
+    def create_volume
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/volumes/create", 'POST')
+      CLI.is_response_success(response)
+      return response
+    end
+
+    def downlowd_updated_data(current_commit)
+
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/downlowd_updated_data", 'POST', {current_commit: current_commit})
+      CLI.is_response_success(response,false)
+      return response
+    end
+    def compare_idx(new_branch, commit=last_local_commit,local_idx=nil)
+      if local_idx.nil?
+        local_idx =  self.generate_idx
+      end
+
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch, current_commit: commit})
+      CLI.is_response_success(response,false)
+      return response
+    end
+
+
+
+    def compare_commits(commit)
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/compare_commits", 'POST', {compare_commit: commit,current_commit:last_local_commit})
+      CLI.is_response_success(response,false)
+      return response
+    end
+
+    def compare_commit(commit)
+        if commit.nil? or commit.empty?
+          commit = last_local_commit
+        end
+        response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
+        CLI.is_response_success(response,false)
+        update_is_new_branch(response["result"]["new_branch"])
+        return response["result"]["new_branch"]
+      end
+
+      def update_idx_with_files_commits!(files, commit_time)
+
+        idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+        files.each do |path|
+          idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
+        end
+        File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
+
+        return true
+      end
+    def update_idx(idx)
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
+
+      return true
+    end
+
+
+      def update_idx_with_commit!(commit)
+        idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+        idx_hash[:commit] = commit
+
+        File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
+        return true
+      end
+
+      def revert(working_dir)
+        FileUtils.rm_rf working_dir
+        # response     = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/revert", 'GET')
+        # CLI.is_response_success(response)
+
+
+      end
+
+    end
+  end

data/lib/cnvrg/experiment.rb

@@ -9,11 +9,11 @@ module Cnvrg
       @slug = nil
     end
 
-    def start(input, platform, machine_name, start_commit, name, email_notification, machine_activity)
+    def start(input, platform, machine_name, start_commit, name, email_notification, machine_activity,script_path)
       res = Cnvrg::API.request(@base_resource + "experiment/start", 'POST',
                                {input: input, platform: platform, machine_name: machine_name, start_commit: start_commit,
-                                title: name, email_notification: email_notification, machine_activity: machine_activity})
-      Cnvrg::CLI.is_response_success(res)
+                                title: name, email_notification: email_notification, machine_activity: machine_activity,script_path:script_path})
+      Cnvrg::CLI.is_response_success(res,false)
 
       @slug = res.to_h["result"].to_h["slug"]
 
@@ -37,7 +37,7 @@ module Cnvrg
     def end_notebook_session(notebook_slug, end_commit)
       res = Cnvrg::API.request(@base_resource + "notebook/end_session", 'POST',
                                {notebook_slug: notebook_slug, end_commit: end_commit})
-      Cnvrg::CLI.is_response_success(res)
+      Cnvrg::CLI.is_response_success(res,false)
 
       return res
 
@@ -78,27 +78,36 @@ module Cnvrg
 
     end
 
-    def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp)
+    def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp, grid,path_to_cmd,data, data_commit)
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
                                                                                                             commit_sha1: commit_to_run,
                                                                                                             instance_type: instance_type,
                                                                                                             scheduling_query:scheduling_query,
-                                                                                                            local_timestamp:local_timestamp})
+                                                                                                            local_timestamp:local_timestamp,
+                                                                                                            grid: grid,
+                                                                                                            path_to_cmd:path_to_cmd,dataset_slug:data,
+                                                                                                            dataset_commit: data_commit})
       return response
     end
 
     def upload_temp_log(temp_log, cpu_average, memory_average)
       response = Cnvrg::API.request(@base_resource + "experiment/upload_temp_log", 'POST', {output: temp_log,
-                                                                                            exp_slug: @slug, cpu_average: cpu_average,
-                                                                                            memory_average: memory_average})
-      Cnvrg::CLI.is_response_success(response)
+                                                                                            exp_slug: @slug})
+      Cnvrg::CLI.is_response_success(response,false)
     end
 
     def end(output, exit_status, end_commit, cpu_average, memory_average)
+      #if remote try to remove
       response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', {output: output, exp_slug: @slug,
                                                                                 exit_status: exit_status, end_commit: end_commit,
                                                                                 cpu_average: cpu_average, memory_average: memory_average})
-      Cnvrg::CLI.is_response_success(response)
+      Cnvrg::CLI.is_response_success(response,false)
+
+      begin
+        FileUtils.rm_rf(["/home/ds/.cnvrg/tmp/exec.log"])
+      rescue
+
+      end
     end
   end
 end

data/lib/cnvrg/files.rb

@@ -1,11 +1,14 @@
 require 'mimemagic'
 require 'aws-sdk'
 require 'URLcrypt'
+require 'tempfile'
 
 module Cnvrg
   class Files
 
-    LARGE_FILE=1024*1024*100
+    LARGE_FILE=1024*1024*5
+    MULTIPART_SPLIT=10000000
+
     attr_reader :base_resource
 
     def initialize(owner, project_slug)
@@ -19,9 +22,11 @@ module Cnvrg
       file_size = File.size(absolute_path).to_f
       mime_type = MimeMagic.by_path(absolute_path)
       content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
+      sha1 = Digest::SHA1.file(absolute_path).hexdigest
+
       upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
                                                                                      commit_sha1: commit_sha1, file_name: file_name,
-                                                                                     file_size: file_size, file_content_type: content_type})
+                                                                                     file_size: file_size, file_content_type: content_type, sha1: sha1})
       if Cnvrg::CLI.is_response_success(upload_resp, false)
         path = upload_resp["result"]["path"]
         if file_size.to_f>= Cnvrg::Files::LARGE_FILE.to_f
@@ -29,24 +34,27 @@ module Cnvrg
         else
           s3_res = upload_small_files_s3(path, absolute_path, content_type)
         end
-        if s3_res
-          Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
-                                                                    blob_id: upload_resp["result"]["id"]})
-          return true
-        end
+        return s3_res
+        # if s3_res
+        # update_s3_resp = Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
+        #                                                           blob_id: upload_resp["result"]["id"]})
+        # is_suc = Cnvrg::CLI.is_response_success(update_s3_resp, false)
+        #
+        # return is_suc
       end
       return false
     end
-    def upload_log_file(absolute_path, relative_path,log_date)
+
+    def upload_log_file(absolute_path, relative_path, log_date)
       file_name = File.basename relative_path
       file_size = File.size(absolute_path).to_f
       content_type = "text/x-log"
       upload_resp = Cnvrg::API.request("/users/#{@owner}/" + "upload_cli_log", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
-                                                                                     file_name: file_name,log_date:log_date,
-                                                                                     file_size: file_size, file_content_type: content_type})
+                                                                                             file_name: file_name, log_date: log_date,
+                                                                                             file_size: file_size, file_content_type: content_type})
       if Cnvrg::CLI.is_response_success(upload_resp, false)
-          path = upload_resp["result"]["path"]
-          s3_res = upload_small_files_s3(path, absolute_path,  "text/plain")
+        path = upload_resp["result"]["path"]
+        s3_res = upload_small_files_s3(path, absolute_path, "text/plain")
       end
       if s3_res
         return true
@@ -54,18 +62,19 @@ module Cnvrg
       return false
 
     end
-    def upload_exec_file(absolute_path,image_name,commit_id)
+
+    def upload_exec_file(absolute_path, image_name, commit_id)
       file_name = File.basename absolute_path
       file_size = File.size(absolute_path).to_f
       content_type = "application/zip"
       begin
         upload_resp = Cnvrg::API.request("users/#{@owner}/images/" + "upload_config", 'POST_FILE', {relative_path: absolute_path,
                                                                                                     file_name: file_name,
-                                                                                                    image_name:image_name,
+                                                                                                    image_name: image_name,
                                                                                                     file_size: file_size,
                                                                                                     file_content_type: content_type,
                                                                                                     project_slug: @project_slug,
-                                                                                                    commit_id:commit_id})
+                                                                                                    commit_id: commit_id})
         # puts upload_resp
         if Cnvrg::CLI.is_response_success(upload_resp, false)
           if upload_resp["result"]["image"] == -1
@@ -75,21 +84,20 @@ module Cnvrg
           s3_res = upload_small_files_s3(path, absolute_path, content_type)
 
         end
-      if s3_res
-        return upload_resp["result"]["id"]
+        if s3_res
+          return upload_resp["result"]["id"]
+        end
+        return false
+      rescue SignalException
+
+        say "\nAborting"
+        exit(1)
       end
-      return false
-    rescue SignalException
 
-      say "\nAborting"
-      exit(1)
     end
 
-  end
 
-
-
-  def upload_image(absolute_path, image_name, owner, is_public, is_base,dpkg,libraries,bash,message,commit_id)
+    def upload_image(absolute_path, image_name, owner, is_public, is_base, dpkg, libraries, bash, message, commit_id)
       file_name = File.basename absolute_path
       file_size = File.size(absolute_path).to_f
       if is_base
@@ -99,40 +107,42 @@ module Cnvrg
         content_type = "application/gzip"
       end
       begin
-      upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload", 'POST_FILE', {relative_path: absolute_path,
-                                                                                          file_name: file_name,
-                                                                                          image_name: image_name,
-                                                                                          file_size: file_size,
-                                                                                          file_content_type: content_type,
-                                                                                          is_public: is_public,
-                                                                                          project_slug: @project_slug,
-                                                                                          commit_id:commit_id ,
-                                                                                          dpkg: dpkg,
-                                                                                          libraries: libraries,
-                                                                                          bash_history: bash,
-                                                                                          commit_message:message,
-                                                                                          is_base: is_base})
-      # puts upload_resp
-      if Cnvrg::CLI.is_response_success(upload_resp, false)
-        path = upload_resp["result"]["path"]
-        s3_res = upload_small_files_s3(path, absolute_path, content_type)
-        if s3_res
-          commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
-          if Cnvrg::CLI.is_response_success(commit_resp, false)
-            return commit_resp["result"]["image"]
-          else
-            return false
-          end
+        upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
+                                                                                                  file_name: file_name,
+                                                                                                  image_name: image_name,
+                                                                                                  file_size: file_size,
+                                                                                                  file_content_type: content_type,
+                                                                                                  is_public: is_public,
+                                                                                                  project_slug: @project_slug,
+                                                                                                  commit_id: commit_id,
+                                                                                                  dpkg: dpkg,
+                                                                                                  py2: libraries,
+                                                                                                  py3: libraries,
+
+                                                                                                  bash_history: bash,
+                                                                                                  commit_message: message,
+                                                                                                  is_base: is_base})
+        # puts upload_resp
+        if Cnvrg::CLI.is_response_success(upload_resp, false)
+          path = upload_resp["result"]["path"]
+          s3_res = upload_small_files_s3(path, absolute_path, content_type)
+          if s3_res
+            commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
+            if Cnvrg::CLI.is_response_success(commit_resp, false)
+              return commit_resp["result"]["image"]
+            else
+              return false
+            end
 
+          end
         end
-      end
-      return false
-      rescue =>e
-        puts e
+        return false
+      rescue => e
       end
 
     end
-    def upload_cnvrg_image(absolute_path, image_name, owner, is_public, is_base,dpkg,libraries,bash,message)
+
+    def upload_cnvrg_image(absolute_path, image_name, owner, is_public, is_base, dpkg, libraries, bash, message)
       file_name = File.basename absolute_path
       file_size = File.size(absolute_path).to_f
       if is_base
@@ -143,16 +153,16 @@ module Cnvrg
       end
       begin
         upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
-                                                                                            file_name: file_name,
-                                                                                            image_name: image_name,
-                                                                                            file_size: file_size,
-                                                                                            file_content_type: content_type,
-                                                                                            is_public: is_public,
-                                                                                            dpkg: dpkg,
-                                                                                            libraries: libraries,
-                                                                                            bash_history: bash,
-                                                                                            commit_message:message,
-                                                                                            is_base: is_base})
+                                                                                                  file_name: file_name,
+                                                                                                  image_name: image_name,
+                                                                                                  file_size: file_size,
+                                                                                                  file_content_type: content_type,
+                                                                                                  is_public: is_public,
+                                                                                                  dpkg: dpkg,
+                                                                                                  libraries: libraries,
+                                                                                                  bash_history: bash,
+                                                                                                  commit_message: message,
+                                                                                                  is_base: is_base})
         # puts upload_resp
         if Cnvrg::CLI.is_response_success(upload_resp, false)
           path = upload_resp["result"]["path"]
@@ -168,8 +178,7 @@ module Cnvrg
           end
         end
         return false
-      rescue =>e
-        puts e
+      rescue => e
       end
 
     end
@@ -188,7 +197,6 @@ module Cnvrg
 
           return true
         rescue => e
-          puts e
           return false
         end
 
@@ -202,174 +210,260 @@ module Cnvrg
 
     def upload_large_files_s3(upload_resp, file_path)
       begin
-        sts_path = upload_resp["result"]["path_sts"]
-        uri = URI.parse(sts_path)
-        http_object = Net::HTTP.new(uri.host, uri.port)
-        http_object.use_ssl = true if uri.scheme == 'https'
-        request = Net::HTTP::Get.new(sts_path)
-
-        body = ""
-        http_object.start do |http|
-          response = http.request request
-          body = response.read_body
+        # speed_uri = URI.parse("https://s3-us-west-2.amazonaws.com/cnvrgw2-dev/speedtest-cli")
+        # speed_http_object = Net::HTTP.new(speed_uri.host, speed_uri.port)
+        # speed_http_object.use_ssl = true if speed_uri.scheme == 'https'
+        # speed_request = Net::HTTP::Get.new("https://s3-us-west-2.amazonaws.com/cnvrgw2-dev/speedtest-cli")
+        #
+        # speed_body = ""
+        # speed_http_object.start do |http|
+        #   response = http.request speed_request
+        #   speed_body = response.read_body
+        # end
+        # speed_tmp = Tempfile.new('speed')
+        # speed_tmp << speed_body
+        # speed_tmp.flush
+        # speed_tmp.close
+        # count = 0
+        # sum = 0
+        # speed_res = `python #{speed_tmp.path} --json`
+        # if !speed_res.nil? and !speed_res.empty?
+        #   upload = JSON.parse(speed_res)["upload"]
+        #   if !upload.nil?
+        #     up_spped = (upload/1000000).round(2)
+        #   end
+        #
+        #
+        #   file_size = File.size(file_path)/1048576
+        #   est_up = ((file_size*8)/up_spped)/60
+        #   puts est_up
+        # end
+
+
+          sts_path = upload_resp["result"]["path_sts"]
+          s4cmd_path = upload_resp["result"]["path_s4cmd"]
+
+          uri = URI.parse(sts_path)
+          http_object = Net::HTTP.new(uri.host, uri.port)
+          http_object.use_ssl = true if uri.scheme == 'https'
+          request = Net::HTTP::Get.new(sts_path)
+
+          body = ""
+          http_object.start do |http|
+            response = http.request request
+            body = response.read_body
+          end
+
+          URLcrypt::key = [body].pack('H*')
+          is_python = false
+          s4cmd_suc = false
+          s4cmd_install_suc = false
+          # python_version=`python --version  > /dev/null 2>&1`; is_python=$?.success?
+          # if is_python
+          #
+          #   s4cmd=`pip freeze 2>/dev/null |grep -e s4cmd -e boto3 > /dev/null 2>&1`; s4cmd_suc=$?.success?
+          #   if !s4cmd_suc
+          #     `pip install s4cmd  > /dev/null 2>&1`; s4cmd_install_suc=$?.success?
+          #   end
+          #
+          # end
+          if !s4cmd_suc and !s4cmd_install_suc
+            s3 = Aws::S3::Resource.new(
+                :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
+                :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
+                :session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
+                :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
+            resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
+                object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
+                upload_file(file_path, {:use_accelerate_endpoint => true})
+          else
+            s4cmd_uri = URI.parse(s4cmd_path)
+            s4cmd_http_object = Net::HTTP.new(s4cmd_uri.host, s4cmd_uri.port)
+            s4cmd_http_object.use_ssl = true if s4cmd_uri.scheme == 'https'
+            s4cmd_request = Net::HTTP::Get.new(s4cmd_path)
+
+            s4cmd_body = ""
+            s4cmd_http_object.start do |http|
+              response = http.request s4cmd_request
+              s4cmd_body = response.read_body
+            end
+            s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
+                                      aws_access_key_id=aws_access_key_id,
+                                      aws_secret_access_key=aws_secret_access_key)", " self.client = self.boto3.client('s3',
+                                      aws_access_key_id='#{ URLcrypt.decrypt(upload_resp["result"]["sts_a"])}',
+                                      aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["sts_s"])}',
+                                      aws_session_token='#{URLcrypt.decrypt(upload_resp["result"]["sts_st"])}')")
+
+
+            tmp = Tempfile.new('s4cmd.py')
+            tmp << s4cmd_new_body
+            tmp.flush
+            tmp.close
+            #
+            is_success = false
+            count = 0
+            while !is_success and count <3
+              resp = `python #{tmp.path} --num-threads=8 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)} > /dev/null 2>&1 `
+              is_success =$?.success?
+              count +=1
+
+            end
+            resp= is_success
+
+          end
+
+          return resp
+
+          rescue => e
+          if File.exist? tmp
+            FileUtils.rm_rf [tmp]
+          end
+          return false
+
         end
-        URLcrypt::key = [body].pack('H*')
-        s3 = Aws::S3::Resource.new(
-            :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
-            :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
-            :session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
-            :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
-        resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
-            object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
-            upload_file(file_path)
-        return resp
-      rescue =>e
-        puts e
-        return false
+        return true
 
       end
-      return true
 
-    end
+      def upload_small_files_s3(url_path, file_path, content_type)
+        url = URI.parse(url_path)
+        file = File.open(file_path, "rb")
+        body = file.read
+        begin
+          Net::HTTP.start(url.host) do |http|
+            http.send_request("PUT", url.request_uri, body, {
+                "content-type" => content_type,
+            })
+          end
+          return true
+        rescue Interrupt
+          return false
+        rescue
+          return false
+        end
+      end
 
-    def upload_small_files_s3(url_path, file_path, content_type)
-      url = URI.parse(url_path)
-      file = File.open(file_path, "rb")
-      body = file.read
-      begin
-        Net::HTTP.start(url.host) do |http|
-          http.send_request("PUT", url.request_uri, body, {
-              "content-type" => content_type,
-          })
+      def upload_url(file_path)
+        response = Cnvrg::API.request(@base_resource + "upload_url", 'POST', {file_s3_path: file_path})
+        if Cnvrg::CLI.is_response_success(response, false)
+          return response
+        else
+          return nil
         end
-        return true
-      rescue Interrupt
-        return false
-      rescue
-        return false
+
       end
-    end
 
-    def upload_url(file_path)
-      response = Cnvrg::API.request(@base_resource + "upload_url", 'POST', {file_s3_path: file_path})
-      if Cnvrg::CLI.is_response_success(response, false)
-        return response
-      else
-        return nil
+      def delete_file(absolute_path, relative_path, commit_sha1)
+        response = Cnvrg::API.request(@base_resource + "delete_file", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
+        return Cnvrg::CLI.is_response_success(response, false)
       end
 
-    end
+      def delete_dir(absolute_path, relative_path, commit_sha1)
+        response = Cnvrg::API.request(@base_resource + "delete_dir", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
+        return Cnvrg::CLI.is_response_success(response, false)
+      end
 
-    def delete_file(absolute_path, relative_path, commit_sha1)
-      response = Cnvrg::API.request(@base_resource + "delete_file", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
-      return Cnvrg::CLI.is_response_success(response, false)
-    end
+      def create_dir(absolute_path, relative_path, commit_sha1)
+        response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
+        return Cnvrg::CLI.is_response_success(response, false)
+      end
 
-    def delete_dir(absolute_path, relative_path, commit_sha1)
-      response = Cnvrg::API.request(@base_resource + "delete_dir", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
-      return Cnvrg::CLI.is_response_success(response, false)
-    end
+      def download_file_s3(absolute_path, relative_path, project_home, commit_sha1=nil, conflict=false)
+        begin
+          res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
+          Cnvrg::CLI.is_response_success(res, false)
+          if res["result"]
+            download_resp = res
+            filename = download_resp["result"]["filename"]
+
+            absolute_path += ".conflict" if conflict
+            sts_path = download_resp["result"]["path_sts"]
+            uri = URI.parse(sts_path)
+            http_object = Net::HTTP.new(uri.host, uri.port)
+            http_object.use_ssl = true if uri.scheme == 'https'
+            request = Net::HTTP::Get.new(sts_path)
+
+            body = ""
+            http_object.start do |http|
+              response = http.request request
+              body = response.read_body
+            end
+            URLcrypt::key = [body].pack('H*')
+            s3 = Aws::S3::Client.new(
+                :access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
+                :secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
+                :session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
+                :region => URLcrypt.decrypt(download_resp["result"]["region"]))
+            File.open(project_home+"/"+absolute_path, 'wb') do |file|
+              resp = s3.get_object({bucket: URLcrypt.decrypt(download_resp["result"]["bucket"]),
+                                    key: URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
+            end
+            return true
+          end
 
-    def create_dir(absolute_path, relative_path, commit_sha1)
-      response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
-      return Cnvrg::CLI.is_response_success(response, false)
-    end
-    def download_file_s3(absolute_path, relative_path, project_home, conflict=false)
-      begin
+        rescue => e
+          return false
+
+        end
+      end
+
+      def download_file(absolute_path, relative_path, project_home, conflict=false)
         res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
         Cnvrg::CLI.is_response_success(res, false)
         if res["result"]
-          download_resp = res
-          filename = download_resp["result"]["filename"]
-
-          absolute_path += ".conflict" if conflict
-          sts_path = download_resp["result"]["path_sts"]
-        uri = URI.parse(sts_path)
-        http_object = Net::HTTP.new(uri.host, uri.port)
-        http_object.use_ssl = true if uri.scheme == 'https'
-        request = Net::HTTP::Get.new(sts_path)
-
-        body = ""
-        http_object.start do |http|
-          response = http.request request
-          body = response.read_body
-        end
-        URLcrypt::key = [body].pack('H*')
-        s3 = Aws::S3::Client.new(
-            :access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
-            :secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
-            :session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
-            :region => URLcrypt.decrypt(download_resp["result"]["region"]))
-
-        File.open(absolute_path, 'wb') do |file|
-          resp = s3.get_object({ bucket:URLcrypt.decrypt(download_resp["result"]["bucket"]),
-                                 key:URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
-        end
-          return true
-        end
+          res = res["result"]
+          return false if res["link"].empty? or res["filename"].empty?
+          filename = res["filename"]
+          file_location = absolute_path.gsub(/#{filename}\/?$/, "")
 
-      rescue =>e
-        puts e
-        return false
+          FileUtils.mkdir_p project_home + "/" + file_location
+          filename += ".conflict" if conflict
 
-      end
-    end
-    def download_file(absolute_path, relative_path, project_home, conflict=false)
-      res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
-      Cnvrg::CLI.is_response_success(res, false)
-      if res["result"]
-        res = res["result"]
-        return false if res["link"].empty? or res["filename"].empty?
-        filename = res["filename"]
-        file_location = absolute_path.gsub(/#{filename}\/?$/, "")
-
-        FileUtils.mkdir_p project_home + "/" + file_location
-        filename += ".conflict" if conflict
-
-        File.open("#{project_home}/#{file_location}/#{filename}", "wb") do |file|
-          file.write open(res["link"]).read
+          File.open("#{project_home}/#{file_location}/#{filename}", "wb") do |file|
+            file.write open(res["link"]).read
+          end
+        else
+          return false
         end
-      else
-        return false
+        return true
       end
-      return true
-    end
 
-    def download_dir(absolute_path, relative_path, project_home)
-      FileUtils.mkdir_p("#{project_home}/#{absolute_path}")
-    end
-    def revoke_download_dir(absolute_path, relative_path, project_home)
-      puts FileUtils.rmtree("#{absolute_path}")
-    end
+      def download_dir(absolute_path, relative_path, project_home)
+        FileUtils.mkdir_p("#{project_home}/#{absolute_path}")
+      end
 
-    def revoke_download_file(project_home,absolute_path,filename,conflict=false)
-      begin
-      file_location = absolute_path.gsub(/#{filename}\/?$/, "")
+      def revoke_download_dir(absolute_path, relative_path, project_home)
+        puts FileUtils.rmtree("#{absolute_path}")
+      end
 
-      filename += ".conflict" if conflict
-      FileUtils.remove("#{file_location}/#{filename}")
-        return true
-      rescue
-       return false
+      def revoke_download_file(project_home, absolute_path, filename, conflict=false)
+        begin
+          file_location = absolute_path.gsub(/#{filename}\/?$/, "")
+
+          filename += ".conflict" if conflict
+          FileUtils.remove("#{file_location}/#{filename}")
+          return true
+        rescue
+          return false
+        end
       end
-    end
 
-    def start_commit(new_branch)
+      def start_commit(new_branch)
 
-      response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug,new_branch:new_branch,
-                                                                              username: @owner})
-      Cnvrg::CLI.is_response_success(response)
-      return response
-    end
+        response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug, new_branch: new_branch,
+                                                                                username: @owner})
+        Cnvrg::CLI.is_response_success(response,false)
+        return response
+      end
 
-    def end_commit(commit_sha1)
-      response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1})
-      return response
-    end
+      def end_commit(commit_sha1)
+        response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1})
+        return response
+      end
 
-    def rollback_commit(commit_sha1)
-      response = Cnvrg::API.request("#{base_resource}/commit/rollback", 'POST', {commit_sha1: commit_sha1})
-      Cnvrg::CLI.is_response_success(response, false)
+      def rollback_commit(commit_sha1)
+        response = Cnvrg::API.request("#{base_resource}/commit/rollback", 'POST', {commit_sha1: commit_sha1})
+        Cnvrg::CLI.is_response_success(response, false)
+      end
     end
   end
-end