cnvrg 0.0.150000 → 0.0.1520000

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 27ff325c9bedde1af4df6c092ec091a39f726285
-  data.tar.gz: 526fcd8e2a4ba9494895677dda0aca2b40a6ed6a
+  metadata.gz: 8350e6b16c815de7c51ea0020bec8e8ee3f760f4
+  data.tar.gz: 45a2c3cec4e0fa3450b6674a8f554c210db57af7
 SHA512:
-  metadata.gz: f104f1192956125b983bd13f13474deff621be3a0e510d45ea5c2cfecf2758af3286cc5600236f8b547845c117b1d9710a4af7578bb0c05b1e1e3171d55403e6
-  data.tar.gz: f97e77a186fd5122cbf9b808d62aaab51f515159104f6fb4c28067b5947d77dbdacadcd50f85bb63264c788c8b10b60c11c27d3b916d9c36489e55e582717723
+  metadata.gz: cb635cb17695ed08f6ee86a175ff18ed5e43fcad39a21f0b3037566605e212c3ff7ec32f535e128e58ba51d9ac446c69d90a9c57450bdede057ebfb48b02aff3
+  data.tar.gz: 20e44b71140893d5aeca636e9f985b92f66471596d9d83a44196f263ffbed2caae9532ffa645aeb5f20158140cadc771bf659201dc91fa9766aa06f7f5eddd3d

data/lib/cnvrg/api.rb

@@ -60,7 +60,6 @@ module Cnvrg
               when 'POST'
                 conn.options.timeout = 420
                 conn.options.open_timeout =420
-
                 response = conn.post "#{endpoint_uri}/#{resource}", data
 
                 if parse_request == true

data/lib/cnvrg/cli.rb

@@ -1151,6 +1151,7 @@ module Cnvrg
 
     end
 
+
     desc 'data upload', 'push data'
     method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
@@ -1237,8 +1238,9 @@ module Cnvrg
             exit(1)
           end
           say "Uploading data", Thor::Shell::Color::BLUE
+          log_file = "#{home_dir}/.cnvrg/tmp/upload_#{File.basename(tar_path)}.log"
           res = false
-            res = @files.upload_tar_file(tar_path, tar_path, commit_sha1)
+          res = @files.upload_tar_file(tar_path, tar_path, commit_sha1)
 
           if res
             say "Commiting data", Thor::Shell::Color::BLUE
@@ -1257,12 +1259,19 @@ module Cnvrg
             end
 
           else
+            if File.exist? log_file
+             @files.upload_data_log_file(log_file,log_file,commit_sha1)
+            end
+
+
             FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
             FileUtils.rm_rf([tar_path]) if File.exist? tar_path
 
 
             @files.rollback_commit(commit_sha1)
             say "Can't upload, Rolling Back all changes.", Thor::Shell::Color::RED
+            say "Error log: #{log_file}", Thor::Shell::Color::RED
+
             exit(1)
           end
 
@@ -1274,6 +1283,9 @@ module Cnvrg
           log_end(-1)
           FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
           FileUtils.rm_rf([tar_path]) if File.exist? tar_path
+          if File.exist? log_file
+            @files.upload_data_log_file(log_file,log_file,commit_sha1)
+          end
 
 
           @files.rollback_commit(commit_sha1)
@@ -1285,39 +1297,26 @@ module Cnvrg
             say "Seems there is no internet connection", Thor::Shell::Color::RED
 
           end
+          if File.exist? log_file
+            @files.upload_data_log_file(log_file,log_file,commit_sha1)
+          end
           FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
           FileUtils.rm_rf([tar_path]) if File.exist? tar_path
 
           log_end(-1, e.message)
           @files.rollback_commit(commit_sha1)
           say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
-          exit(0)
-        end
-        if verbose
-          say "#{check} Done", Thor::Shell::Color::BLUE
-          if successful_updates.size >0
-            say "Updated:", Thor::Shell::Color::GREEN
-            suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
-            say suc.join("\n"), Thor::Shell::Color::GREEN
-          end
-          if successful_deletions.size >0
-            say "Deleted:", Thor::Shell::Color::GREEN
-            del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
-            say del.join("\n"), Thor::Shell::Color::GREEN
-          end
+          say "Error log: #{log_file}", Thor::Shell::Color::RED
 
 
-          say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
-        else
-          say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
-
+          exit(0)
         end
+        say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
+
 
 
       rescue => e
         log_end(-1, e.message)
-
-
         say "Error occurd, \nAborting", Thor::Shell::Color::RED
         @files.rollback_commit(commit_sha1)
         exit(1)
@@ -2299,32 +2298,15 @@ module Cnvrg
                 begin
                   stdout.each do |line|
                     cur_time = Time.now
-                    if remote
-                      stats = usage_metrics_in_docker(docker_id)
-                      cpu = stats[0]
-                      memory = stats[1]
-                    else
-                      memory = memory_usage()
-                      cpu = cpu_usage()
-                    end
-
-                    memory_total << memory.to_f
-                    cpu_total << cpu.to_f
                     real_time= Time.now-real
 
                     cur_log = { time: cur_time,
                                message: line,
                                type: "stdout",
-                               real: real_time,
-                                cpu: cpu,
-                                memory: memory
+                               real: real_time
+
                     }
-                    if is_on_gpu
-                      gpu_stats = gpu_util
-                      gpu_utilization = gpu_stats[0]
-                      gpu_memory_util = gpu_stats[1]
-                      cur_log.merge!(gpu_util:gpu_utilization,gpu_memory_util:gpu_memory_util)
-                    end
+
 
                       if print_log
                       puts cur_log
@@ -2334,6 +2316,28 @@ module Cnvrg
                     begin
                       if time_to_upload !=0
                         if time_to_upload <= Time.now - start_loop
+                          if remote
+                            stats = usage_metrics_in_docker(docker_id)
+                            cpu = stats[0]
+                            memory = stats[1]
+                            if  is_on_gpu
+                              gpu_stats = gpu_util
+                              gpu_utilization = gpu_stats[0]
+                              gpu_memory_util = gpu_stats[1]
+                            end
+                          else
+                            memory = memory_usage()
+                            cpu = cpu_usage()
+                          end
+                          log.each do |l|
+                            if remote and is_on_gpu
+                              l.merge!(cpu:cpu,memory:memory, gpu_util:gpu_utilization,gpu_memory_util:gpu_memory_util)
+                            else
+                              l.merge!(cpu:cpu,memory:memory)
+                            end
+                          end
+
+
                           #upload current log
                           # cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
                           # memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
@@ -4749,10 +4753,25 @@ module Cnvrg
 
           end
           $LOG = LogStashLogger.new(type: :file, path: logfile, sync: true)
-        rescue => e
+          remove_old_log_files()
         end
       end
 
+      def remove_old_log_files()
+        begin
+        last_week = (Time.now - (7*24*60*60)).strftime("%Y-%m-%d")
+        home = File.expand_path('~')
+        log_files  = Dir["#{home}/.cnvrg/tmp/*.log"]
+        log_files.each do |l|
+          if File.mtime(l).strftime("%Y-%m-%d") < last_week
+            FileUtils.rm_rf(l)
+          end
+        end
+        end
+
+      end
+
+
 
       def verify_logged_in(in_dir=true)
         log_handler()
@@ -5119,7 +5138,7 @@ module Cnvrg
           if docker_id.nil?
             docker_id = `cat /etc/hostname`
           end
-          stats = `sudo docker stats #{docker_id} --no-stream --format "{{.CPUPerc}},{{.MemPerc}},{{.BlockIO}}"`
+            stats = `sudo docker stats #{docker_id} --no-stream --format "{{.CPUPerc}},{{.MemPerc}},{{.BlockIO}}"`
           if !stats.nil?
             conv = stats.split(",")
             cpu = conv[0].gsub!("%","").to_f

data/lib/cnvrg/datafiles.rb

@@ -22,10 +22,9 @@ module Cnvrg
       mime_type = MimeMagic.by_path(absolute_path)
       content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
       sha1 = Digest::SHA1.file(absolute_path).hexdigest
-
       upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
                                                                                      commit_sha1: commit_sha1, file_name: file_name,
-                                                                                     file_size: file_size, file_content_type: content_type,sha1:sha1})
+                                                                                     file_size: file_size, file_content_type: content_type, sha1: sha1})
       if Cnvrg::CLI.is_response_success(upload_resp, false)
         path = upload_resp["result"]["path"]
         if file_size.to_f>= Cnvrg::Files::LARGE_FILE.to_f
@@ -41,44 +40,77 @@ module Cnvrg
       end
       return false
     end
+
     def upload_tar_file(absolute_path, relative_path, commit_sha1)
       begin
-      file_name = File.basename relative_path
-      file_size = File.size(absolute_path).to_f
-      mime_type = MimeMagic.by_path(absolute_path)
-      content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
-      sha1 = Digest::SHA1.file(absolute_path).hexdigest
+        file_name = File.basename relative_path
+        file_size = File.size(absolute_path).to_f
+        mime_type = MimeMagic.by_path(absolute_path)
+        content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
+        begin
+          chunked_bytes = [100, (file_size*0.01)].min
+          total_yanked = ""
+          open(absolute_path, "rb") do |f|
+            total_yanked = f.read(chunked_bytes)
+          end
+          if !total_yanked.empty?
+            sha1 = Digest::SHA1.hexdigest(total_yanked)
+          else
+            sha1 = Digest::SHA1.file(absolute_path).hexdigest
+          end
+        rescue
+          sha1 = Digest::SHA1.file(absolute_path).hexdigest
+        end
 
-      upload_resp = Cnvrg::API.request(@base_resource + "upload_tar_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
-                                                                                     commit_sha1: commit_sha1, file_name: file_name,
-                                                                                     file_size: file_size, file_content_type: content_type,sha1:sha1})
-      if Cnvrg::CLI.is_response_success(upload_resp, false)
-        path = upload_resp["result"]["path"]
-        s3_res = upload_large_files_s3(upload_resp, absolute_path)
-        if s3_res
-          Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
-                                                                    blob_id: upload_resp["result"]["id"]})
-          return true
+        upload_resp = Cnvrg::API.request(@base_resource + "upload_tar_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
+                                                                                           commit_sha1: commit_sha1, file_name: file_name,
+                                                                                           file_size: file_size, file_content_type: content_type, sha1: sha1})
+        if Cnvrg::CLI.is_response_success(upload_resp, false)
+          path = upload_resp["result"]["path"]
+          s3_res = upload_large_files_s3(upload_resp, absolute_path)
+          if s3_res
+            Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
+                                                                      blob_id: upload_resp["result"]["id"]})
+            return true
+          end
+        else
+          return false
         end
-      else
-        return false
-      end
       rescue => e
         puts e.message
         return false
       end
 
     end
-    def upload_log_file(absolute_path, relative_path,log_date)
+
+    def upload_log_file(absolute_path, relative_path, log_date)
       file_name = File.basename relative_path
       file_size = File.size(absolute_path).to_f
       content_type = "text/x-log"
       upload_resp = Cnvrg::API.request("/users/#{@owner}/" + "upload_cli_log", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
-                                                                                     file_name: file_name,log_date:log_date,
-                                                                                     file_size: file_size, file_content_type: content_type})
+                                                                                             file_name: file_name, log_date: log_date,
+                                                                                             file_size: file_size, file_content_type: content_type})
       if Cnvrg::CLI.is_response_success(upload_resp, false)
-          path = upload_resp["result"]["path"]
-          s3_res = upload_small_files_s3(path, absolute_path,  "text/plain")
+        path = upload_resp["result"]["path"]
+        s3_res = upload_small_files_s3(path, absolute_path, "text/plain")
+      end
+      if s3_res
+        return true
+      end
+      return false
+
+    end
+    def upload_data_log_file(absolute_path, relative_path,data_commit_sha)
+      file_name = File.basename relative_path
+      file_size = File.size(absolute_path).to_f
+      content_type = "text/x-log"
+      upload_resp = Cnvrg::API.request("/users/#{@owner}/" + "upload_data_log", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
+                                                                                             file_name: file_name, log_date: Time.now,
+                                                                                             file_size: file_size, file_content_type: content_type,
+                                                                                              data_commit_sha1:data_commit_sha})
+      if Cnvrg::CLI.is_response_success(upload_resp, false)
+        path = upload_resp["result"]["path"]
+        s3_res = upload_small_files_s3(path, absolute_path, "text/plain")
       end
       if s3_res
         return true
@@ -86,18 +118,20 @@ module Cnvrg
       return false
 
     end
-    def upload_exec_file(absolute_path,image_name,commit_id)
+
+
+    def upload_exec_file(absolute_path, image_name, commit_id)
       file_name = File.basename absolute_path
       file_size = File.size(absolute_path).to_f
       content_type = "application/zip"
       begin
         upload_resp = Cnvrg::API.request("users/#{@owner}/images/" + "upload_config", 'POST_FILE', {relative_path: absolute_path,
                                                                                                     file_name: file_name,
-                                                                                                    image_name:image_name,
+                                                                                                    image_name: image_name,
                                                                                                     file_size: file_size,
                                                                                                     file_content_type: content_type,
                                                                                                     project_slug: @project_slug,
-                                                                                                    commit_id:commit_id})
+                                                                                                    commit_id: commit_id})
         # puts upload_resp
         if Cnvrg::CLI.is_response_success(upload_resp, false)
           if upload_resp["result"]["image"] == -1
@@ -107,21 +141,20 @@ module Cnvrg
           s3_res = upload_small_files_s3(path, absolute_path, content_type)
 
         end
-      if s3_res
-        return upload_resp["result"]["id"]
+        if s3_res
+          return upload_resp["result"]["id"]
+        end
+        return false
+      rescue SignalException
+
+        say "\nAborting"
+        exit(1)
       end
-      return false
-    rescue SignalException
 
-      say "\nAborting"
-      exit(1)
     end
 
-  end
-
-
 
-  def upload_image(absolute_path, image_name, owner, is_public, is_base,dpkg,libraries,bash,message,commit_id)
+    def upload_image(absolute_path, image_name, owner, is_public, is_base, dpkg, libraries, bash, message, commit_id)
       file_name = File.basename absolute_path
       file_size = File.size(absolute_path).to_f
       if is_base
@@ -131,41 +164,42 @@ module Cnvrg
         content_type = "application/gzip"
       end
       begin
-      upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
-                                                                                          file_name: file_name,
-                                                                                          image_name: image_name,
-                                                                                          file_size: file_size,
-                                                                                          file_content_type: content_type,
-                                                                                          is_public: is_public,
-                                                                                          project_slug: @project_slug,
-                                                                                          commit_id:commit_id ,
-                                                                                          dpkg: dpkg,
-                                                                                          py2: libraries,
-                                                                                          py3: libraries,
-
-                                                                                          bash_history: bash,
-                                                                                          commit_message:message,
-                                                                                          is_base: is_base})
-      # puts upload_resp
-      if Cnvrg::CLI.is_response_success(upload_resp, false)
-        path = upload_resp["result"]["path"]
-        s3_res = upload_small_files_s3(path, absolute_path, content_type)
-        if s3_res
-          commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
-          if Cnvrg::CLI.is_response_success(commit_resp, false)
-            return commit_resp["result"]["image"]
-          else
-            return false
-          end
+        upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
+                                                                                                  file_name: file_name,
+                                                                                                  image_name: image_name,
+                                                                                                  file_size: file_size,
+                                                                                                  file_content_type: content_type,
+                                                                                                  is_public: is_public,
+                                                                                                  project_slug: @project_slug,
+                                                                                                  commit_id: commit_id,
+                                                                                                  dpkg: dpkg,
+                                                                                                  py2: libraries,
+                                                                                                  py3: libraries,
+
+                                                                                                  bash_history: bash,
+                                                                                                  commit_message: message,
+                                                                                                  is_base: is_base})
+        # puts upload_resp
+        if Cnvrg::CLI.is_response_success(upload_resp, false)
+          path = upload_resp["result"]["path"]
+          s3_res = upload_small_files_s3(path, absolute_path, content_type)
+          if s3_res
+            commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
+            if Cnvrg::CLI.is_response_success(commit_resp, false)
+              return commit_resp["result"]["image"]
+            else
+              return false
+            end
 
+          end
         end
-      end
-      return false
-      rescue =>e
+        return false
+      rescue => e
       end
 
     end
-    def upload_cnvrg_image(absolute_path, image_name, owner, is_public, is_base,dpkg,libraries,bash,message)
+
+    def upload_cnvrg_image(absolute_path, image_name, owner, is_public, is_base, dpkg, libraries, bash, message)
       file_name = File.basename absolute_path
       file_size = File.size(absolute_path).to_f
       if is_base
@@ -176,16 +210,16 @@ module Cnvrg
       end
       begin
         upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
-                                                                                            file_name: file_name,
-                                                                                            image_name: image_name,
-                                                                                            file_size: file_size,
-                                                                                            file_content_type: content_type,
-                                                                                            is_public: is_public,
-                                                                                            dpkg: dpkg,
-                                                                                            libraries: libraries,
-                                                                                            bash_history: bash,
-                                                                                            commit_message:message,
-                                                                                            is_base: is_base})
+                                                                                                  file_name: file_name,
+                                                                                                  image_name: image_name,
+                                                                                                  file_size: file_size,
+                                                                                                  file_content_type: content_type,
+                                                                                                  is_public: is_public,
+                                                                                                  dpkg: dpkg,
+                                                                                                  libraries: libraries,
+                                                                                                  bash_history: bash,
+                                                                                                  commit_message: message,
+                                                                                                  is_base: is_base})
         # puts upload_resp
         if Cnvrg::CLI.is_response_success(upload_resp, false)
           path = upload_resp["result"]["path"]
@@ -201,7 +235,7 @@ module Cnvrg
           end
         end
         return false
-      rescue =>e
+      rescue => e
       end
 
     end
@@ -231,7 +265,7 @@ module Cnvrg
 
     end
 
-    def   upload_large_files_s3(upload_resp, file_path)
+    def upload_large_files_s3(upload_resp, file_path)
       begin
         sts_path = upload_resp["result"]["path_sts"]
         # s4cmd_path = upload_resp["result"]["path_s4cmd"]
@@ -246,76 +280,62 @@ module Cnvrg
           response = http.request request
           body = response.read_body
         end
-        is_python = false
-        s4cmd_suc = false
-        s4cmd_install_suc = false
+        home_dir = File.expand_path('~')
+        log_file = "#{home_dir}/.cnvrg/tmp/upload_#{File.basename(file_path)}.log"
 
         URLcrypt::key = [body].pack('H*')
-
-        # python_version=`python --version  > /dev/null 2>&1` ;  is_python=$?.success?
-        # if is_python
+        client = Aws::S3::Client.new(:access_key_id => URLcrypt.decrypt(upload_resp["result"]["new_a"]),
+                                     :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["new_s"]),
+                                     :region => URLcrypt.decrypt(upload_resp["result"]["region"]),
+                                     :logger => Logger.new(log_file),
+                                     :http_open_timeout => 60, :retry_limit => 20,
+                                     :http_wire_trace => true)
+        s3 = Aws::S3::Resource.new(client: client)
+
+        resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
+            object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
+            upload_file(file_path, {:use_accelerate_endpoint => true})
+        # s4cmd_path = upload_resp["result"]["path_s4cmd"]
         #
-        #   s4cmd=`pip freeze 2>/dev/null |grep -e s4cmd -e boto3 > /dev/null 2>&1` ;  s4cmd_suc=$?.success?
-        #   if !s4cmd_suc
-        #     `pip install s4cmd  > /dev/null 2>&1` ; s4cmd_install_suc=$?.success?
-        #   end
+        # s4cmd_uri = URI.parse(s4cmd_path)
+        # s4cmd_http_object = Net::HTTP.new(s4cmd_uri.host, s4cmd_uri.port)
+        # s4cmd_http_object.use_ssl = true if s4cmd_uri.scheme == 'https'
+        # s4cmd_request = Net::HTTP::Get.new(s4cmd_path)
         #
+        # s4cmd_body = ""
+        # s4cmd_http_object.start do |http|
+        #   response = http.request s4cmd_request
+        #   s4cmd_body = response.read_body
         # end
-        resp = false
-
-        if  !s4cmd_suc and !s4cmd_install_suc
-          s3 = Aws::S3::Resource.new(
-              :access_key_id => URLcrypt.decrypt(upload_resp["result"]["new_a"]),
-              :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["new_s"]),
-              :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
-          resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
-              object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
-              upload_file(file_path,{:use_accelerate_endpoint=>true})
-
-        # else
-
-          # s4cmd_uri = URI.parse(s4cmd_path)
-          # s4cmd_http_object = Net::HTTP.new(s4cmd_uri.host, s4cmd_uri.port)
-          # s4cmd_http_object.use_ssl = true if s4cmd_uri.scheme == 'https'
-          # s4cmd_request = Net::HTTP::Get.new(s4cmd_path)
-          #
-          # s4cmd_body = ""
-          # s4cmd_http_object.start do |http|
-          #   response = http.request s4cmd_request
-          #   s4cmd_body = response.read_body
-          # end
-          #
-          # s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
-          #                             aws_access_key_id=aws_access_key_id,
-          #                             aws_secret_access_key=aws_secret_access_key)"," self.client = self.boto3.client('s3',
-          #                             aws_access_key_id='#{ URLcrypt.decrypt(upload_resp["result"]["sts_a"])}',
-          #                             aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["sts_s"])}',
-          #                             aws_session_token='#{URLcrypt.decrypt(upload_resp["result"]["sts_st"])}')")
-          #
-          # tmp = Tempfile.new('s4cmd.py')
-          # tmp << s4cmd_new_body
-          # tmp.flush
-          # tmp.close
-          #
-          #   is_success = false
-          # count = 0
-          # while !is_success and count <3
-          #   resp = `python #{tmp.path} --num-threads=8 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)}  > /dev/null 2>&1`
-          #   is_success =$?.success?
-          #   count +=1
-          #
-          # end
-          # resp= is_success
-
+        # s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
+        #                               aws_access_key_id=aws_access_key_id,
+        #                               aws_secret_access_key=aws_secret_access_key)", " self.client = self.boto3.client('s3',
+        #                               aws_access_key_id='#{ URLcrypt.decrypt(upload_resp["result"]["new_a"])}',
+        #                               aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["new_s"])}')")
+        #
+        #
+        # tmp = Tempfile.new('s4cmd.py')
+        # tmp << s4cmd_new_body
+        # tmp.flush
+        # tmp.close
+        # #
+        # is_success = false
+        # count = 0
+        # while !is_success and count <3
+        #   resp = `python #{tmp.path} --num-threads=8 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)} > /dev/null 2>&1 `
+        #   is_success =$?.success?
+        #   count +=1
+        #
+        # end
+        # resp= is_success
+        if resp == true
+          FileUtils.rm_rf(log_file)
         end
 
         return resp
 
       rescue => e
         puts e.message
-        # if File.exist? tmp
-        #   FileUtils.rm_rf [tmp]
-        # end
         return false
 
       end
@@ -366,9 +386,10 @@ module Cnvrg
       response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
       return Cnvrg::CLI.is_response_success(response, false)
     end
-    def download_file_s3(absolute_path, relative_path, project_home, conflict=false,commit_sha1=nil)
+
+    def download_file_s3(absolute_path, relative_path, project_home, conflict=false, commit_sha1=nil)
       begin
-        res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path,commit_sha1:commit_sha1})
+        res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
         Cnvrg::CLI.is_response_success(res, false)
         if res["result"]
           download_resp = res
@@ -376,38 +397,38 @@ module Cnvrg
 
           absolute_path += ".conflict" if conflict
           sts_path = download_resp["result"]["path_sts"]
-        uri = URI.parse(sts_path)
-        http_object = Net::HTTP.new(uri.host, uri.port)
-        http_object.use_ssl = true if uri.scheme == 'https'
-        request = Net::HTTP::Get.new(sts_path)
+          uri = URI.parse(sts_path)
+          http_object = Net::HTTP.new(uri.host, uri.port)
+          http_object.use_ssl = true if uri.scheme == 'https'
+          request = Net::HTTP::Get.new(sts_path)
 
-        body = ""
-        http_object.start do |http|
-          response = http.request request
-          body = response.read_body
-        end
-        URLcrypt::key = [body].pack('H*')
-        s3 = Aws::S3::Client.new(
-            :access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
-            :secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
-            :session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
-            :region => URLcrypt.decrypt(download_resp["result"]["region"]))
-        File.open(project_home+"/"+absolute_path, 'wb') do |file|
-          resp = s3.get_object({ bucket:URLcrypt.decrypt(download_resp["result"]["bucket"]),
-                                 key:URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
-        end
+          body = ""
+          http_object.start do |http|
+            response = http.request request
+            body = response.read_body
+          end
+          URLcrypt::key = [body].pack('H*')
+          s3 = Aws::S3::Client.new(
+              :access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
+              :secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
+              :session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
+              :region => URLcrypt.decrypt(download_resp["result"]["region"]))
+          File.open(project_home+"/"+absolute_path, 'wb') do |file|
+            resp = s3.get_object({bucket: URLcrypt.decrypt(download_resp["result"]["bucket"]),
+                                  key: URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
+          end
           return true
         end
 
-      rescue =>e
+      rescue => e
         return false
 
       end
     end
 
-    def download_data_file(commit_sha1,dataset_home)
+    def download_data_file(commit_sha1, dataset_home)
       begin
-        res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1:commit_sha1})
+        res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1: commit_sha1})
         Cnvrg::CLI.is_response_success(res, false)
         if res["result"]
           download_resp = res
@@ -432,13 +453,13 @@ module Cnvrg
               :region => URLcrypt.decrypt(download_resp["result"]["region"]))
 
           File.open(dataset_home+"/"+filename, 'wb') do |file|
-            resp = s3.get_object({ bucket:URLcrypt.decrypt(download_resp["result"]["bucket"]),
-                                   key:URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
+            resp = s3.get_object({bucket: URLcrypt.decrypt(download_resp["result"]["bucket"]),
+                                  key: URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
           end
           return filename
         end
 
-      rescue =>e
+      rescue => e
         return false
 
       end
@@ -468,27 +489,28 @@ module Cnvrg
     def download_dir(dataset_home, absolute_path)
       FileUtils.mkdir_p("#{dataset_home}/#{absolute_path}")
     end
+
     def revoke_download_dir(absolute_path)
       puts FileUtils.rmtree("#{absolute_path}")
     end
 
-    def revoke_download_file(absolute_path,filename,conflict=false)
+    def revoke_download_file(absolute_path, filename, conflict=false)
       begin
-      file_location = absolute_path.gsub(/#{filename}\/?$/, "")
+        file_location = absolute_path.gsub(/#{filename}\/?$/, "")
 
-      filename += ".conflict" if conflict
-      FileUtils.remove("#{file_location}/#{filename}")
+        filename += ".conflict" if conflict
+        FileUtils.remove("#{file_location}/#{filename}")
         return true
       rescue
-       return false
+        return false
       end
     end
 
     def start_commit(new_branch)
 
-      response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug,new_branch:false,
+      response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: false,
                                                                               username: @owner})
-      Cnvrg::CLI.is_response_success(response,false)
+      Cnvrg::CLI.is_response_success(response, false)
       return response
     end
 
@@ -496,8 +518,9 @@ module Cnvrg
       response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1})
       return response
     end
-    def end_commit_tar(commit_sha1,cur_idx)
-      response = Cnvrg::API.request("#{base_resource}/commit/end_tar", 'POST', {commit_sha1: commit_sha1,idx: cur_idx})
+
+    def end_commit_tar(commit_sha1, cur_idx)
+      response = Cnvrg::API.request("#{base_resource}/commit/end_tar", 'POST', {commit_sha1: commit_sha1, idx: cur_idx})
       return response
     end
 

data/lib/cnvrg/dataset.rb

@@ -116,7 +116,7 @@ module Cnvrg
           File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
           File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
         rescue => e
-          "Exception in init data: #{e.message}"
+          "Exception in init data: #"
           return false
         end
         return true

data/lib/cnvrg/version.rb

@@ -1,4 +1,6 @@
 module Cnvrg
-    VERSION = '0.0.150000'
+    VERSION = '0.0.1520000'
+
+
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.0.150000
+  version: 0.0.1520000
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-16 00:00:00.000000000 Z
+date: 2017-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -388,9 +388,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.
 test_files: []
-has_rdoc: