cnvrg 2.1.6 → 2.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Readme.md +20 -1
- data/cnvrg.gemspec +1 -0
- data/lib/cnvrg/cli.rb +8 -12
- data/lib/cnvrg/downloader/clients/azure_client.rb +22 -4
- data/lib/cnvrg/downloader/clients/gcp_client.rb +1 -1
- data/lib/cnvrg/helpers/agent.rb +3 -1
- data/lib/cnvrg/helpers.rb +0 -42
- data/lib/cnvrg/version.rb +1 -1
- data/lib/cnvrg.rb +4 -0
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 850c099310fb739283b4fe34c0c0eccfb9bd4297e9d525daa497eb45481ea707
|
4
|
+
data.tar.gz: d84a2854fe5ede6ab5a65804d11068a00c88dccf5d341a9d54728c3f5870a9b0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0e3da1a344367dacf8496c0e59de07770ce69db0ab59009d14fd27bc15ae6e9e56bb653e49c95d732a5b1dde962a550cba8da3310806c615015fa693ae9899d3
|
7
|
+
data.tar.gz: 01eda251528f076aa244558272ef4a630145fb7e8752f5f25b7c1f6bfb39be61e33189ad1e8b5e50ac1d80825de303ec479186eb65b0db6d2b62cce1b693c99a
|
data/Readme.md
CHANGED
@@ -98,4 +98,23 @@
|
|
98
98
|
* DEV-14633 - Bug: End sync did not complete, causing the experiment to get stuck in "terminating"
|
99
99
|
## Version v2.1.6
|
100
100
|
2022-08-09
|
101
|
-
* DEV-14682 - Bug: git-Walki: CLI/SDK experiments goes into debug mode for Github+SSH integrated projects
|
101
|
+
* DEV-14682 - Bug: git-Walki: CLI/SDK experiments goes into debug mode for Github+SSH integrated projects
|
102
|
+
## Version v2.1.7
|
103
|
+
2022-08-24
|
104
|
+
* DEV-15229 - Bug: CLI - warnings from faraday gem when running any CLI command
|
105
|
+
## Version v2.1.8
|
106
|
+
2022-09-04
|
107
|
+
* DEV-15473 - Bug: CLI - Errors while cloning a project that has file containing spaces in their names
|
108
|
+
## Version v2.1.9
|
109
|
+
2022-09-06
|
110
|
+
* DEV-15423 - Bug: Workspace - Jupyter process gets killed
|
111
|
+
* DEV-15451 - Bug: CLI - sync error "undefined method `encode' for nil:NilClass" on GCP storage
|
112
|
+
## Version v2.1.10
|
113
|
+
2022-10-26
|
114
|
+
* DEV-15858 - Bug: Tensorboard compare - ongoing experiments' artifacts aren't cloned to session
|
115
|
+
## Version v2.1.11
|
116
|
+
2022-11-03
|
117
|
+
* DEV-16090 - Bug: Tensorboard compare - webapp gets stuck in "init" state
|
118
|
+
## Version v2.1.12
|
119
|
+
2022-11-09
|
120
|
+
* DEV-15972 - Bug: Customer cannot use dataset query due to ssl error
|
data/cnvrg.gemspec
CHANGED
@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
|
|
28
28
|
spec.add_runtime_dependency 'ffi', '~> 1.9', '>= 1.9.10'
|
29
29
|
spec.add_runtime_dependency 'mimemagic', '~> 0.3.1', '>=0.3.7'
|
30
30
|
spec.add_runtime_dependency 'faraday', '~> 0.15.2'
|
31
|
+
spec.add_runtime_dependency 'warning', '~> 1.3.0'
|
31
32
|
spec.add_runtime_dependency 'netrc', '~> 0.11.0'
|
32
33
|
spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
|
33
34
|
spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -1019,7 +1019,7 @@ module Cnvrg
|
|
1019
1019
|
begin
|
1020
1020
|
unless File.exist?(fullpath)
|
1021
1021
|
downloader.safe_operation("#{abs_path}/#{file_name}") do
|
1022
|
-
download = open(f["url"])
|
1022
|
+
download = open(f["url"],{ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE})
|
1023
1023
|
IO.copy_stream(download, fullpath)
|
1024
1024
|
end
|
1025
1025
|
end
|
@@ -5003,32 +5003,28 @@ module Cnvrg
|
|
5003
5003
|
|
5004
5004
|
log_message("compare is running")
|
5005
5005
|
while true
|
5006
|
-
log_message("
|
5006
|
+
log_message("Comparing the following experiment slugs: #{slugs}")
|
5007
5007
|
slugs.each do |exp_slug|
|
5008
5008
|
begin
|
5009
5009
|
if exps_map[exp_slug].blank?
|
5010
5010
|
exp = @project.get_experiment(exp_slug)["experiment"]
|
5011
5011
|
else
|
5012
5012
|
exp = exps_map[exp_slug]
|
5013
|
-
log_message("#{exp["title"]}
|
5013
|
+
log_message("Experiment '#{exp["title"]}' end commit already cloned, skipping it", Thor::Shell::Color::BLUE)
|
5014
5014
|
next
|
5015
5015
|
end
|
5016
5016
|
exp_name = exp["title"]
|
5017
5017
|
if exp["end_commit"].present? and exp["status"] != "Ongoing"
|
5018
|
-
log_message("#{exp_name} has ended, getting files from end commit", Thor::Shell::Color::BLUE)
|
5018
|
+
log_message("Experiment '#{exp_name}' has ended, getting files from its end commit", Thor::Shell::Color::BLUE)
|
5019
5019
|
num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project)
|
5020
5020
|
exps_map[exp_slug] = exp
|
5021
5021
|
else
|
5022
|
-
log_message("#{exp_name} is running
|
5023
|
-
|
5024
|
-
|
5025
|
-
log_message("Failed to get kube files, using last commit", Thor::Shell::Color::BLUE)
|
5026
|
-
num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"]["sha1"])
|
5027
|
-
copied_commits << exp["last_successful_commit"]["sha1"]
|
5028
|
-
end
|
5022
|
+
log_message("Experiment '#{exp_name}' is running, getting files from its last successful commit", Thor::Shell::Color::BLUE)
|
5023
|
+
num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"]["sha1"])
|
5024
|
+
copied_commits << exp["last_successful_commit"]["sha1"]
|
5029
5025
|
end
|
5030
5026
|
|
5031
|
-
log_message("New
|
5027
|
+
log_message("New .tfevent files downloaded", Thor::Shell::Color::BLUE) if num_of_new_files > 0
|
5032
5028
|
rescue => e
|
5033
5029
|
Cnvrg::Logger.log_error(e)
|
5034
5030
|
end
|
@@ -15,15 +15,13 @@ module Cnvrg
|
|
15
15
|
|
16
16
|
def download(storage_path, local_path, decrypt: true)
|
17
17
|
prepare_download(local_path)
|
18
|
-
|
19
18
|
storage_path = Cnvrg::Helpers.decrypt(@key, @iv, storage_path) if decrypt
|
20
19
|
|
21
20
|
# We generate a temp uri in order to stream the file instead of using "get_blob" that overflows memory
|
22
21
|
uri = client.send(:blob_uri, @container, storage_path)
|
23
22
|
|
24
|
-
generator = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(@account_name, @access_key)
|
25
23
|
|
26
|
-
expiring_url =
|
24
|
+
expiring_url = self.signed_uri_custom(
|
27
25
|
uri,
|
28
26
|
false,
|
29
27
|
service: 'b',
|
@@ -32,12 +30,32 @@ module Cnvrg
|
|
32
30
|
start: (Time.now - (5 * 60)).utc.iso8601, # start 5 minutes ago
|
33
31
|
expiry: (Time.now + 60 * 60 * 2).utc.iso8601 # expire in 2 hours
|
34
32
|
)
|
35
|
-
|
36
33
|
# Stream the file without loading it all into memory
|
37
34
|
download = open(expiring_url)
|
38
35
|
IO.copy_stream(download, local_path)
|
39
36
|
end
|
40
37
|
|
38
|
+
def signed_uri_custom(uri, use_account_sas, options)
|
39
|
+
# url sent to generate_service_sas_token should be DECODED (file names with spaces should not be encoded with %20)
|
40
|
+
url = URI.decode(uri.path)
|
41
|
+
generator = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(@account_name, @access_key)
|
42
|
+
|
43
|
+
CGI::parse(uri.query || "").inject({}) { |memo, (k, v)| memo[k.to_sym] = v; memo }
|
44
|
+
|
45
|
+
if options[:service] == (nil) && uri.host != (nil)
|
46
|
+
host_splits = uri.host.split(".")
|
47
|
+
options[:service] = host_splits[1].chr if host_splits.length > 1 && host_splits[0] == @account_name
|
48
|
+
end
|
49
|
+
|
50
|
+
sas_params = if use_account_sas
|
51
|
+
generator.generate_account_sas_token(options)
|
52
|
+
else
|
53
|
+
generator.generate_service_sas_token(url, options)
|
54
|
+
end
|
55
|
+
|
56
|
+
URI.parse(uri.to_s + (uri.query.nil? ? "?" : "&") + sas_params)
|
57
|
+
end
|
58
|
+
|
41
59
|
def upload(storage_path, local_path)
|
42
60
|
begin
|
43
61
|
client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))
|
@@ -6,7 +6,7 @@ module Cnvrg
|
|
6
6
|
class GcpClient < Client
|
7
7
|
def initialize(project_id: nil, credentials: nil, bucket_name: nil, sts: nil)
|
8
8
|
@key, @iv = extract_key_iv(sts)
|
9
|
-
@project_id = Cnvrg::Helpers.decrypt(@key, @iv, project_id)
|
9
|
+
@project_id = Cnvrg::Helpers.decrypt(@key, @iv, project_id) if project_id.present?
|
10
10
|
@credentials_path = Cnvrg::Helpers.decrypt(@key, @iv, credentials)
|
11
11
|
@tempfile = nil
|
12
12
|
@bucket_name = Cnvrg::Helpers.decrypt(@key, @iv, bucket_name)
|
data/lib/cnvrg/helpers/agent.rb
CHANGED
@@ -22,6 +22,7 @@ class Cnvrg::Helpers::Agent
|
|
22
22
|
@slug = slug
|
23
23
|
@files_exist = files_exist
|
24
24
|
@container_name = container_name
|
25
|
+
@execute_command_completed = false
|
25
26
|
@is_new_main = !ENV["MAIN_CONTAINER_PORT"].blank?
|
26
27
|
@main_name = @is_new_main ? "main" : "slave"
|
27
28
|
@run_in_main = @container_name.downcase == @main_name
|
@@ -79,6 +80,7 @@ class Cnvrg::Helpers::Agent
|
|
79
80
|
command_status = Status::ABORTED
|
80
81
|
@exit_status = 127
|
81
82
|
end
|
83
|
+
@execute_command_completed = true
|
82
84
|
finish_log = "Command: #{@command} with slug: #{@slug} finished"
|
83
85
|
finish_log += " after #{@real_execution_retries} retries" if @real_execution_retries > 0
|
84
86
|
log_internal(finish_log)
|
@@ -97,7 +99,7 @@ class Cnvrg::Helpers::Agent
|
|
97
99
|
|
98
100
|
def periodic_thread
|
99
101
|
Thread.new do
|
100
|
-
while
|
102
|
+
while !@execute_command_completed
|
101
103
|
Thread.exit if @log_interval.blank?
|
102
104
|
sleep(@log_interval)
|
103
105
|
send_logs
|
data/lib/cnvrg/helpers.rb
CHANGED
@@ -374,48 +374,6 @@ parameters:
|
|
374
374
|
end
|
375
375
|
return num_of_new_files
|
376
376
|
end
|
377
|
-
|
378
|
-
def get_experiment_events_log_via_kubectl(exp, namespace)
|
379
|
-
dest_dir = exp["slug"]
|
380
|
-
result = `kubectl -n #{namespace} get pods | grep #{exp["slug"]}`
|
381
|
-
|
382
|
-
pod_name = result.split(" ")[0]
|
383
|
-
return false, 0 if pod_name.blank?
|
384
|
-
FileUtils.mkdir_p(dest_dir)
|
385
|
-
working_dir = `kubectl -n #{namespace} exec #{pod_name} -c agent -- pwd`
|
386
|
-
working_dir.strip!
|
387
|
-
res = `kubectl -n #{namespace} exec #{pod_name} -c agent -- /bin/bash -c "ls -R #{working_dir}"`
|
388
|
-
files_and_folders = res.split("\n\n")
|
389
|
-
all_files = []
|
390
|
-
|
391
|
-
files_and_folders.each do |file_and_folder|
|
392
|
-
files = file_and_folder.split("\n")
|
393
|
-
if files.first.include?(":")
|
394
|
-
folder = files.first.gsub(":", "")
|
395
|
-
|
396
|
-
folder = folder.sub(working_dir + "/", "")
|
397
|
-
files = files.drop(1)
|
398
|
-
end
|
399
|
-
files.each do |file|
|
400
|
-
if file.include?("tfevents")
|
401
|
-
all_files << "#{folder}/#{file}"
|
402
|
-
end
|
403
|
-
end
|
404
|
-
end
|
405
|
-
|
406
|
-
num_of_new_files = 0
|
407
|
-
all_files.each do |file|
|
408
|
-
file_dir = "#{dest_dir}/#{File.dirname(file)}"
|
409
|
-
FileUtils.mkdir_p(file_dir)
|
410
|
-
num_of_new_files += 1 unless File.exist?("#{dest_dir}/#{file}")
|
411
|
-
res = `kubectl -n #{namespace} cp #{pod_name}:#{file} -c agent #{dest_dir}/#{file}`
|
412
|
-
end
|
413
|
-
|
414
|
-
return true, num_of_new_files
|
415
|
-
rescue => e
|
416
|
-
Cnvrg::Logger.log_error(e)
|
417
|
-
return false, 0
|
418
|
-
end
|
419
377
|
end
|
420
378
|
|
421
379
|
end
|
data/lib/cnvrg/version.rb
CHANGED
data/lib/cnvrg.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.1.
|
4
|
+
version: 2.1.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2022-
|
13
|
+
date: 2022-11-30 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: bundler
|
@@ -150,6 +150,20 @@ dependencies:
|
|
150
150
|
- - "~>"
|
151
151
|
- !ruby/object:Gem::Version
|
152
152
|
version: 0.15.2
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: warning
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - "~>"
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: 1.3.0
|
160
|
+
type: :runtime
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - "~>"
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: 1.3.0
|
153
167
|
- !ruby/object:Gem::Dependency
|
154
168
|
name: netrc
|
155
169
|
requirement: !ruby/object:Gem::Requirement
|