cnvrg 0.8.2 → 0.8.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/datafiles.rb +8 -3
- data/lib/cnvrg/experiment.rb +1 -1
- data/lib/cnvrg/files.rb +24 -4
- data/lib/cnvrg/helpers.rb +8 -1
- data/lib/cnvrg/project.rb +2 -3
- data/lib/cnvrg/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c13496f69eb2a70e01466fec1a111235b78af234
|
4
|
+
data.tar.gz: 74454b3cefaff038466b3bb2bd65824a05a39aea
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8bbb9a27f946777237dd2bf6008e1f5a4083af1a9f78cda649e7d205b01e015dfaa2ee0d855406ae589dbb56c0528c19f5b6c9de7b3a13d1685f51abd51d8f6b
|
7
|
+
data.tar.gz: 5baf19679c532c8207985d455de91569e2d5524c6e2f4904b930257ecc2971c8b90a7b7a6f5aaade8fea54c74ecb015537db6e7164f77c30b4b7562e451fc424
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -431,7 +431,8 @@ module Cnvrg
|
|
431
431
|
region = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["region"])
|
432
432
|
|
433
433
|
bucket = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["bucket"])
|
434
|
-
|
434
|
+
server_side_encryption =upload_resp["result"]["server_side_encryption"]
|
435
|
+
use_accelerate_endpoint = false
|
435
436
|
if is_s3 or is_s3.nil?
|
436
437
|
|
437
438
|
use_accelerate_endpoint =true
|
@@ -453,7 +454,11 @@ module Cnvrg
|
|
453
454
|
end
|
454
455
|
|
455
456
|
|
456
|
-
|
457
|
+
if server_side_encryption.present? and server_side_encryption == 'false'
|
458
|
+
options = {:use_accelerate_endpoint => use_accelerate_endpoint}
|
459
|
+
else
|
460
|
+
options = {:use_accelerate_endpoint => use_accelerate_endpoint, :server_side_encryption => server_side_encryption}
|
461
|
+
end
|
457
462
|
|
458
463
|
|
459
464
|
|
@@ -462,7 +467,7 @@ module Cnvrg
|
|
462
467
|
|
463
468
|
resp = s3.bucket(bucket).
|
464
469
|
object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
|
465
|
-
upload_file(file_path,
|
470
|
+
upload_file(file_path,options)
|
466
471
|
|
467
472
|
|
468
473
|
return resp
|
data/lib/cnvrg/experiment.rb
CHANGED
@@ -77,7 +77,7 @@ module Cnvrg
|
|
77
77
|
end
|
78
78
|
def get_machine_activity(working_dir)
|
79
79
|
begin
|
80
|
-
machine_activity = File.open("
|
80
|
+
machine_activity = File.open("#{working_dir}/.cnvrg/machine_activity", "rb").read
|
81
81
|
machine_activity = machine_activity.to_s.strip
|
82
82
|
ma_id = machine_activity.to_i
|
83
83
|
return ma_id
|
data/lib/cnvrg/files.rb
CHANGED
@@ -39,8 +39,9 @@ module Cnvrg
|
|
39
39
|
end
|
40
40
|
|
41
41
|
def upload_files_old(files_list, commit_sha1, progress: nil)
|
42
|
-
Parallel.map(files_list) do |file|
|
43
|
-
|
42
|
+
# Parallel.map(files_list) do |file|
|
43
|
+
files_list.each do |file|
|
44
|
+
Cnvrg::Helpers.try_until_success{self.upload_old("#{@project_home}/#{file}", file, commit_sha1)}
|
44
45
|
progress.progress += 1
|
45
46
|
end
|
46
47
|
end
|
@@ -50,6 +51,7 @@ module Cnvrg
|
|
50
51
|
Cnvrg::Logger.log_info("Uploading project files")
|
51
52
|
return if files_list.blank?
|
52
53
|
if Cnvrg::Helpers.server_version < 1
|
54
|
+
Cnvrg::Logger.log_info("Upload files to older server..")
|
53
55
|
return self.upload_files_old(files_list, commit_sha1, progress: progress)
|
54
56
|
end
|
55
57
|
files_list = files_list.map{|x| [x,self.parse_file(x)]}.to_h
|
@@ -133,6 +135,14 @@ module Cnvrg
|
|
133
135
|
{relative_path: file, absolute_path: abs_path, file_name: file_name, file_size: file_size, content_type: content_type, sha1: sha1}
|
134
136
|
end
|
135
137
|
|
138
|
+
def upload_old(absolute_path, relative_path, commit_sha1)
|
139
|
+
if relative_path.ends_with? '/'
|
140
|
+
self.create_dir(absolute_path, relative_path, commit_sha1)
|
141
|
+
else
|
142
|
+
self.upload_file(absolute_path, relative_path, commit_sha1)
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
136
146
|
def upload_file(absolute_path, relative_path, commit_sha1)
|
137
147
|
file_name = File.basename relative_path
|
138
148
|
file_size = File.size(absolute_path).to_f
|
@@ -452,6 +462,9 @@ module Cnvrg
|
|
452
462
|
|
453
463
|
bucket = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["bucket"])
|
454
464
|
is_s3 = upload_resp["result"]["is_s3"]
|
465
|
+
server_side_encryption =upload_resp["result"]["server_side_encryption"]
|
466
|
+
use_accelerate_endpoint = false
|
467
|
+
|
455
468
|
if is_s3 or is_s3.nil?
|
456
469
|
use_accelerate_endpoint =true
|
457
470
|
client = Aws::S3::Client.new(
|
@@ -462,7 +475,6 @@ module Cnvrg
|
|
462
475
|
:http_open_timeout => 60, :retry_limit => 20)
|
463
476
|
else
|
464
477
|
endpoint = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["endpoint"])
|
465
|
-
use_accelerate_endpoint = false
|
466
478
|
client = Aws::S3::Client.new(
|
467
479
|
:access_key_id =>access,
|
468
480
|
:secret_access_key => secret,
|
@@ -470,10 +482,16 @@ module Cnvrg
|
|
470
482
|
:endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
|
471
483
|
:http_open_timeout => 60, :retry_limit => 20)
|
472
484
|
end
|
485
|
+
|
486
|
+
if !server_side_encryption.nil? and !server_side_encryption
|
487
|
+
options = {:use_accelerate_endpoint => use_accelerate_endpoint}
|
488
|
+
else
|
489
|
+
options = {:use_accelerate_endpoint => use_accelerate_endpoint, :server_side_encryption => server_side_encryption}
|
490
|
+
end
|
473
491
|
s3 = Aws::S3::Resource.new(client: client)
|
474
492
|
resp = s3.bucket(bucket).
|
475
493
|
object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
|
476
|
-
upload_file(file_path,
|
494
|
+
upload_file(file_path, options)
|
477
495
|
|
478
496
|
return resp
|
479
497
|
|
@@ -596,6 +614,7 @@ module Cnvrg
|
|
596
614
|
bucket = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
|
597
615
|
file_key = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
|
598
616
|
|
617
|
+
|
599
618
|
is_s3 = download_resp["result"]["is_s3"]
|
600
619
|
if is_s3 or is_s3.nil?
|
601
620
|
client = Aws::S3::Client.new(
|
@@ -906,6 +925,7 @@ module Cnvrg
|
|
906
925
|
Cnvrg::CLI.log_message("Failed #{file}")
|
907
926
|
end
|
908
927
|
rescue => e
|
928
|
+
Cnvrg::Logger.log_error(e)
|
909
929
|
raise Exception.new("Cant upload files")
|
910
930
|
end
|
911
931
|
end
|
data/lib/cnvrg/helpers.rb
CHANGED
@@ -309,6 +309,8 @@ parameters:
|
|
309
309
|
|
310
310
|
bucket = Cnvrg::Helpers.decrypt(key,iv, files["bucket"])
|
311
311
|
is_s3 = files["is_s3"]
|
312
|
+
server_side_encryption =files["server_side_encryption"]
|
313
|
+
|
312
314
|
if is_s3 or is_s3.nil?
|
313
315
|
client = Aws::S3::Client.new(
|
314
316
|
:access_key_id =>access,
|
@@ -328,7 +330,12 @@ parameters:
|
|
328
330
|
:http_open_timeout => 60, :retry_limit => 20)
|
329
331
|
use_accelerate_endpoint = false
|
330
332
|
end
|
331
|
-
|
333
|
+
|
334
|
+
if server_side_encryption.present? and server_side_encryption == 'false'
|
335
|
+
upload_options = {:use_accelerate_endpoint => use_accelerate_endpoint}
|
336
|
+
else
|
337
|
+
upload_options = {:use_accelerate_endpoint => use_accelerate_endpoint, :server_side_encryption => server_side_encryption}
|
338
|
+
end
|
332
339
|
return {client: client, key: key, iv: iv, bucket: bucket, upload_options: upload_options}
|
333
340
|
end
|
334
341
|
|
data/lib/cnvrg/project.rb
CHANGED
@@ -325,6 +325,7 @@ module Cnvrg
|
|
325
325
|
File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write new_config.to_yaml }
|
326
326
|
end
|
327
327
|
def generate_output_dir(output_dir)
|
328
|
+
Cnvrg::Logger.log_info("Generating output dir for #{output_dir}")
|
328
329
|
upload_list = []
|
329
330
|
list = Dir.glob("#{output_dir}/**/*", File::FNM_DOTMATCH)
|
330
331
|
Parallel.map(list, in_threads: IDXParallelThreads) do |e|
|
@@ -339,10 +340,8 @@ module Cnvrg
|
|
339
340
|
if Dir.exists? output_dir
|
340
341
|
upload_list << output_dir + "/"
|
341
342
|
end
|
342
|
-
|
343
|
-
|
343
|
+
Cnvrg::Logger.log_info("Uploading: #{upload_list.join(", ")}")
|
344
344
|
return upload_list
|
345
|
-
|
346
345
|
end
|
347
346
|
def generate_output_dir_tmp(output_dir)
|
348
347
|
upload_list = []
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-
|
12
|
+
date: 2018-11-01 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|