cnvrg 0.8.2 → 0.8.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0513fe7eb80b8633e3012478fe86baad4f11585f
-  data.tar.gz: 7a31042c326555ffc8d59c14abc2c09c0635a138
+  metadata.gz: c13496f69eb2a70e01466fec1a111235b78af234
+  data.tar.gz: 74454b3cefaff038466b3bb2bd65824a05a39aea
 SHA512:
-  metadata.gz: c811b17a9f6f77ef8c99363adbd0868686349f93bcae165845b3a8c9acb280d2a6cc11d1f15e31280b6aa1fc22bcbe7cb94af4316234a1aafa54b04ba3b46f62
-  data.tar.gz: f59236f39b298e2ab1be29202c58e9bdc36aec4a6f9b8f413251c87d7ad90cf300a620dcc6395a9e728c7c479770dad5a546c74a7f67b99461a4efad4e863b6c
+  metadata.gz: 8bbb9a27f946777237dd2bf6008e1f5a4083af1a9f78cda649e7d205b01e015dfaa2ee0d855406ae589dbb56c0528c19f5b6c9de7b3a13d1685f51abd51d8f6b
+  data.tar.gz: 5baf19679c532c8207985d455de91569e2d5524c6e2f4904b930257ecc2971c8b90a7b7a6f5aaade8fea54c74ecb015537db6e7164f77c30b4b7562e451fc424

data/lib/cnvrg/datafiles.rb

@@ -431,7 +431,8 @@ module Cnvrg
         region =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["region"])
 
         bucket =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["bucket"])
-
+        server_side_encryption =upload_resp["result"]["server_side_encryption"]
+        use_accelerate_endpoint = false
         if is_s3 or is_s3.nil?
 
           use_accelerate_endpoint =true
@@ -453,7 +454,11 @@ module Cnvrg
         end
 
 
-
+        if server_side_encryption.present? and server_side_encryption  == 'false'
+          options = {:use_accelerate_endpoint => use_accelerate_endpoint}
+        else
+          options = {:use_accelerate_endpoint => use_accelerate_endpoint, :server_side_encryption => server_side_encryption}
+        end
 
 
 
@@ -462,7 +467,7 @@ module Cnvrg
 
         resp = s3.bucket(bucket).
             object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
-            upload_file(file_path, {:use_accelerate_endpoint => use_accelerate_endpoint,:server_side_encryption => 'AES256'})
+            upload_file(file_path,options)
 
 
         return resp

data/lib/cnvrg/experiment.rb

@@ -77,7 +77,7 @@ module Cnvrg
     end
     def get_machine_activity(working_dir)
       begin
-        machine_activity = File.open("/home/ds/notebooks/.cnvrg/machine_activity", "rb").read
+        machine_activity = File.open("#{working_dir}/.cnvrg/machine_activity", "rb").read
         machine_activity = machine_activity.to_s.strip
         ma_id = machine_activity.to_i
         return ma_id

data/lib/cnvrg/files.rb

@@ -39,8 +39,9 @@ module Cnvrg
     end
 
     def upload_files_old(files_list, commit_sha1, progress: nil)
-      Parallel.map(files_list) do |file|
-        Cnvrg::Helpers.try_until_success{self.upload_file("#{@project_home}/#{file}", file, commit_sha1)}
+      # Parallel.map(files_list) do |file|
+      files_list.each do |file|
+        Cnvrg::Helpers.try_until_success{self.upload_old("#{@project_home}/#{file}", file, commit_sha1)}
         progress.progress += 1
       end
     end
@@ -50,6 +51,7 @@ module Cnvrg
       Cnvrg::Logger.log_info("Uploading project files")
       return if files_list.blank?
       if Cnvrg::Helpers.server_version < 1
+        Cnvrg::Logger.log_info("Upload files to older server..")
         return self.upload_files_old(files_list, commit_sha1, progress: progress)
       end
       files_list = files_list.map{|x| [x,self.parse_file(x)]}.to_h
@@ -133,6 +135,14 @@ module Cnvrg
       {relative_path: file, absolute_path: abs_path, file_name: file_name, file_size: file_size, content_type: content_type, sha1: sha1}
     end
 
+    def upload_old(absolute_path, relative_path, commit_sha1)
+      if relative_path.ends_with? '/'
+        self.create_dir(absolute_path, relative_path, commit_sha1)
+      else
+        self.upload_file(absolute_path, relative_path, commit_sha1)
+      end
+    end
+
     def upload_file(absolute_path, relative_path, commit_sha1)
       file_name = File.basename relative_path
       file_size = File.size(absolute_path).to_f
@@ -452,6 +462,9 @@ module Cnvrg
 
           bucket =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["bucket"])
           is_s3 = upload_resp["result"]["is_s3"]
+        server_side_encryption =upload_resp["result"]["server_side_encryption"]
+        use_accelerate_endpoint = false
+
           if is_s3 or is_s3.nil?
             use_accelerate_endpoint =true
           client = Aws::S3::Client.new(
@@ -462,7 +475,6 @@ module Cnvrg
           :http_open_timeout => 60, :retry_limit => 20)
           else
             endpoint = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["endpoint"])
-            use_accelerate_endpoint = false
             client = Aws::S3::Client.new(
                 :access_key_id =>access,
                 :secret_access_key => secret,
@@ -470,10 +482,16 @@ module Cnvrg
                 :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
                 :http_open_timeout => 60, :retry_limit => 20)
           end
+
+        if !server_side_encryption.nil? and !server_side_encryption
+          options = {:use_accelerate_endpoint => use_accelerate_endpoint}
+        else
+          options = {:use_accelerate_endpoint => use_accelerate_endpoint, :server_side_encryption => server_side_encryption}
+        end
             s3 = Aws::S3::Resource.new(client: client)
             resp = s3.bucket(bucket).
                 object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
-                upload_file(file_path, {:use_accelerate_endpoint => use_accelerate_endpoint, :server_side_encryption => 'AES256'})
+                upload_file(file_path, options)
 
           return resp
 
@@ -596,6 +614,7 @@ module Cnvrg
             bucket =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
             file_key =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
 
+
             is_s3 = download_resp["result"]["is_s3"]
             if is_s3 or is_s3.nil?
               client = Aws::S3::Client.new(
@@ -906,6 +925,7 @@ module Cnvrg
           Cnvrg::CLI.log_message("Failed #{file}")
         end
       rescue => e
+        Cnvrg::Logger.log_error(e)
         raise Exception.new("Cant upload files")
       end
     end

data/lib/cnvrg/helpers.rb

@@ -309,6 +309,8 @@ parameters:
 
       bucket =  Cnvrg::Helpers.decrypt(key,iv, files["bucket"])
       is_s3 = files["is_s3"]
+      server_side_encryption =files["server_side_encryption"]
+
       if is_s3 or is_s3.nil?
         client = Aws::S3::Client.new(
             :access_key_id =>access,
@@ -328,7 +330,12 @@ parameters:
             :http_open_timeout => 60, :retry_limit => 20)
         use_accelerate_endpoint = false
       end
-      upload_options = {:use_accelerate_endpoint => use_accelerate_endpoint,:server_side_encryption => 'AES256'}
+
+      if server_side_encryption.present? and server_side_encryption  == 'false'
+        upload_options = {:use_accelerate_endpoint => use_accelerate_endpoint}
+      else
+        upload_options = {:use_accelerate_endpoint => use_accelerate_endpoint, :server_side_encryption => server_side_encryption}
+      end
       return {client: client, key: key, iv: iv, bucket: bucket, upload_options: upload_options}
     end
 

data/lib/cnvrg/project.rb

@@ -325,6 +325,7 @@ module Cnvrg
       File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write new_config.to_yaml }
     end
     def generate_output_dir(output_dir)
+      Cnvrg::Logger.log_info("Generating output dir for #{output_dir}")
       upload_list = []
       list = Dir.glob("#{output_dir}/**/*", File::FNM_DOTMATCH)
       Parallel.map(list, in_threads: IDXParallelThreads) do |e|
@@ -339,10 +340,8 @@ module Cnvrg
       if Dir.exists? output_dir
         upload_list << output_dir + "/"
       end
-
-
+      Cnvrg::Logger.log_info("Uploading: #{upload_list.join(", ")}")
       return upload_list
-
     end
     def generate_output_dir_tmp(output_dir)
       upload_list = []

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '0.8.2'
+  VERSION = '0.8.4'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.8.4
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-23 00:00:00.000000000 Z
+date: 2018-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler