cnvrg 1.5.9.1 → 1.5.9.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +22 -10
- data/lib/cnvrg/data.rb +3 -1
- data/lib/cnvrg/datafiles.rb +10 -8
- data/lib/cnvrg/dataset.rb +5 -0
- data/lib/cnvrg/downloader/client.rb +4 -0
- data/lib/cnvrg/downloader/clients/azure_client.rb +7 -0
- data/lib/cnvrg/downloader/clients/s3_client.rb +0 -5
- data/lib/cnvrg/files.rb +11 -5
- data/lib/cnvrg/helpers.rb +6 -8
- data/lib/cnvrg/helpers/executer.rb +23 -7
- data/lib/cnvrg/image.rb +100 -0
- data/lib/cnvrg/image_cli.rb +25 -0
- data/lib/cnvrg/version.rb +1 -1
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 15c9e9bdcf068b1bcba64ac65c5589ea53d39478cb50e251f4d956eab7e5a40a
|
4
|
+
data.tar.gz: d4b7bd60ace357d13f502771c260c039de02bf4c6b7894510e2db52b6026dc38
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fb8813704368baf2e80422248d948b29bc4189dd703c77f670b3056b446cffdf8867edcd552003eb40cc21fac0b80257faded1e18faeca912733f50bf1aa711b
|
7
|
+
data.tar.gz: f51367399df0f4a8faa3a1d6e7f160b77050c6f4a9eba88ea5963baeaae7eb7f07dde7e06c297c8003c0b0114dcfe6fcce4a3ff8011e5db319188f2789d7a13e
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -29,6 +29,7 @@ require 'cnvrg/project'
|
|
29
29
|
require 'cnvrg/files'
|
30
30
|
require 'cnvrg/experiment'
|
31
31
|
require 'cnvrg/Images'
|
32
|
+
require 'cnvrg/image'
|
32
33
|
require 'cnvrg/dataset'
|
33
34
|
require 'cnvrg/datafiles'
|
34
35
|
require 'cnvrg/data'
|
@@ -40,6 +41,7 @@ require 'cnvrg/org_helpers'
|
|
40
41
|
require 'cnvrg/cli/subcommand'
|
41
42
|
require 'cnvrg/cli/flow'
|
42
43
|
require 'cnvrg/cli/task'
|
44
|
+
require 'cnvrg/image_cli'
|
43
45
|
require 'cnvrg/helpers/executer'
|
44
46
|
require 'cnvrg/downloader/client'
|
45
47
|
require 'cnvrg/downloader/clients/s3_client'
|
@@ -160,7 +162,7 @@ module Cnvrg
|
|
160
162
|
|
161
163
|
class << self
|
162
164
|
# Hackery.Take the run method away from Thor so that we can redefine it.
|
163
|
-
|
165
|
+
|
164
166
|
def is_thor_reserved_word?(word, type)
|
165
167
|
return false if word == "run"
|
166
168
|
super
|
@@ -172,7 +174,11 @@ module Cnvrg
|
|
172
174
|
desc "job", "manage running jobs", :hide => false
|
173
175
|
subcommand "job", JobCli
|
174
176
|
|
175
|
-
|
177
|
+
desc "image [COMMAND]", "build existing images", :hide => true
|
178
|
+
subcommand "image", ImageCli
|
179
|
+
|
180
|
+
|
181
|
+
desc "flow", "mange project flows", :hide => true
|
176
182
|
subcommand "flow", Cnvrg::Commands::Flow
|
177
183
|
|
178
184
|
|
@@ -855,7 +861,7 @@ module Cnvrg
|
|
855
861
|
method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
|
856
862
|
method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
|
857
863
|
|
858
|
-
def clone_data(dataset_url,only_tree=false,commit=nil,query=nil,read=false,remote=false)
|
864
|
+
def clone_data(dataset_url,only_tree=false,commit=nil,query=nil,read=false,remote=false, relative: false)
|
859
865
|
begin
|
860
866
|
verify_logged_in(false)
|
861
867
|
log_start(__method__, args, options)
|
@@ -885,8 +891,9 @@ module Cnvrg
|
|
885
891
|
@files = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
|
886
892
|
log_message("Downloading files", Thor::Shell::Color::BLUE)
|
887
893
|
if @dataset.softlinked?
|
888
|
-
@files.cp_ds
|
889
|
-
|
894
|
+
@files.cp_ds(relative)
|
895
|
+
@executer.set_dataset_status(dataset: @dataset.slug, status: "cloned") if @executer
|
896
|
+
log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
|
890
897
|
@dataset.write_success
|
891
898
|
return
|
892
899
|
end
|
@@ -907,7 +914,7 @@ module Cnvrg
|
|
907
914
|
:total => files_count,
|
908
915
|
:autofinish => true)
|
909
916
|
|
910
|
-
|
917
|
+
|
911
918
|
while files['keys'].length > 0
|
912
919
|
Cnvrg::Logger.log_info("download multiple files, #{downloaded_files.size} files downloaded")
|
913
920
|
@files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read)
|
@@ -1745,6 +1752,10 @@ module Cnvrg
|
|
1745
1752
|
|
1746
1753
|
clone_resp = Project.clone_dir_remote(slug, owner, slug,true)
|
1747
1754
|
idx_status = Project.new(get_project_home).generate_idx
|
1755
|
+
@executer = Cnvrg::Helpers::Executer.get_executer
|
1756
|
+
if @executer.present?
|
1757
|
+
@executer.update_git_commit
|
1758
|
+
end
|
1748
1759
|
end
|
1749
1760
|
|
1750
1761
|
|
@@ -2342,17 +2353,18 @@ module Cnvrg
|
|
2342
2353
|
end
|
2343
2354
|
end
|
2344
2355
|
rescue => e
|
2356
|
+
error_message = "Error occured, #{e.message}\nAborting"
|
2345
2357
|
if e.is_a? SignalException
|
2346
2358
|
say "\nAborting", Thor::Shell::Color::BLUE
|
2347
2359
|
say "\nRolling back all changes", Thor::Shell::Color::BLUE
|
2348
2360
|
else
|
2349
|
-
log_message(
|
2361
|
+
log_message(error_message, Thor::Shell::Color::RED)
|
2350
2362
|
log_error(e)
|
2351
2363
|
end
|
2352
2364
|
@files.rollback_commit(commit_sha1) unless commit_sha1.nil?
|
2353
2365
|
print_res = {
|
2354
2366
|
'success' => "false",
|
2355
|
-
'message' =>
|
2367
|
+
'message' => error_message
|
2356
2368
|
}
|
2357
2369
|
puts "\n"
|
2358
2370
|
puts JSON[print_res] if return_id
|
@@ -2465,7 +2477,7 @@ module Cnvrg
|
|
2465
2477
|
:total => update_total,
|
2466
2478
|
:autofinish => true)
|
2467
2479
|
conflicts = @files.mark_conflicts(result)
|
2468
|
-
|
2480
|
+
|
2469
2481
|
log_message("Found some conflicts, check .conflict files.", Thor::Shell::Color::BLUE) if conflicts > 0
|
2470
2482
|
update_res = @files.download_files_in_chunks(result["updated_on_server"], progress: progressbar) if result["updated_on_server"].present?
|
2471
2483
|
added_res = @files.download_files_in_chunks(result["added"], progress: progressbar) if result["added"].present?
|
@@ -2492,7 +2504,7 @@ module Cnvrg
|
|
2492
2504
|
else
|
2493
2505
|
log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, ((sync or options["sync"]) ? false : true))
|
2494
2506
|
end
|
2495
|
-
return true
|
2507
|
+
return true
|
2496
2508
|
end
|
2497
2509
|
rescue SignalException => e
|
2498
2510
|
Cnvrg::Logger.log_error(e)
|
data/lib/cnvrg/data.rb
CHANGED
@@ -90,15 +90,17 @@ module Cnvrg
|
|
90
90
|
method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
|
91
91
|
method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
|
92
92
|
method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
|
93
|
+
method_option :relative, :type => :boolean, :aliases => ["-rel", "--relative"], :default => false
|
93
94
|
|
94
95
|
def clone(dataset_url)
|
96
|
+
#test
|
95
97
|
cli = Cnvrg::CLI.new()
|
96
98
|
only_tree =options[:only_tree]
|
97
99
|
commit =options[:commit]
|
98
100
|
query =options[:query]
|
99
101
|
read = options[:read]
|
100
102
|
remote = options[:remote]
|
101
|
-
cli.clone_data(dataset_url, only_tree=only_tree,commit=commit, query=query, read=read, remote=remote)
|
103
|
+
cli.clone_data(dataset_url, only_tree=only_tree,commit=commit, query=query, read=read, remote=remote, relative: options[:relative])
|
102
104
|
end
|
103
105
|
|
104
106
|
desc 'data verify DATASETS_TITLES', 'verify datasets', :hide => true
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -70,9 +70,9 @@ module Cnvrg
|
|
70
70
|
end
|
71
71
|
end
|
72
72
|
|
73
|
-
def cp_ds
|
73
|
+
def cp_ds(relative: false)
|
74
74
|
prefix = @dataset.get_dataset["bucket_prefix"]
|
75
|
-
|
75
|
+
batch_size = 10000
|
76
76
|
pbar = ProgressBar.create(:title => "Download Progress",
|
77
77
|
:progress_mark => '=',
|
78
78
|
:format => "%b%i| %c Files downloaded",
|
@@ -83,19 +83,21 @@ module Cnvrg
|
|
83
83
|
in_threads: ParallelThreads,
|
84
84
|
in_processes: Cnvrg::CLI::ParallelProcesses,
|
85
85
|
isolation: true,
|
86
|
-
finish: ->(*args) {pbar.progress += 1}
|
86
|
+
finish: ->(*args) { pbar.progress += 1 }
|
87
87
|
}
|
88
|
-
|
88
|
+
finished = false
|
89
|
+
while not finished
|
90
|
+
current_batch, marker = @downloader.fetch_files(prefix: prefix, marker: marker, limit: batch_size)
|
91
|
+
if marker.blank?
|
92
|
+
finished = true
|
93
|
+
end
|
89
94
|
Parallel.map(current_batch, parallel_options) do |file|
|
90
|
-
#current_batch.map do |file|
|
91
95
|
next if file.end_with? "/"
|
92
|
-
cutted_key =
|
96
|
+
cutted_key = relative ? @downloader.cut_prefix(prefix, file) : file
|
93
97
|
dest_path = File.join(@dataset.local_path, cutted_key)
|
94
98
|
@downloader.download(file, dest_path, decrypt: false)
|
95
99
|
file
|
96
100
|
end
|
97
|
-
marker = current_batch.last
|
98
|
-
current_batch = @downloader.fetch_files(prefix: prefix, marker: marker, limit: 500)
|
99
101
|
end
|
100
102
|
end
|
101
103
|
|
data/lib/cnvrg/dataset.rb
CHANGED
@@ -9,6 +9,7 @@ module Cnvrg
|
|
9
9
|
|
10
10
|
def initialize(project_home = '', dataset_url: '')
|
11
11
|
begin
|
12
|
+
@info = {}
|
12
13
|
if project_home.present?
|
13
14
|
@local_path = project_home
|
14
15
|
@working_dir = project_home
|
@@ -28,6 +29,10 @@ module Cnvrg
|
|
28
29
|
end
|
29
30
|
end
|
30
31
|
|
32
|
+
def soft_linked?
|
33
|
+
@dataset_call["dataset_type"] == "soft_link_dataset"
|
34
|
+
end
|
35
|
+
|
31
36
|
def init_home(remote: false)
|
32
37
|
dataset_home = File.join(Dir.pwd, @slug)
|
33
38
|
if Dir.exists? dataset_home
|
@@ -23,6 +23,13 @@ module Cnvrg
|
|
23
23
|
client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))
|
24
24
|
end
|
25
25
|
|
26
|
+
def fetch_files(prefix: nil, marker: nil, limit: 10000)
|
27
|
+
blobs = client.list_blobs(@container, prefix: prefix, max_results: limit, marker: marker)
|
28
|
+
next_marker = blobs.continuation_token
|
29
|
+
files = blobs.map{|x| x.name}
|
30
|
+
[files, next_marker]
|
31
|
+
end
|
32
|
+
|
26
33
|
|
27
34
|
private
|
28
35
|
def client
|
data/lib/cnvrg/files.rb
CHANGED
@@ -88,16 +88,22 @@ module Cnvrg
|
|
88
88
|
# resolve bucket
|
89
89
|
res = resp['result']
|
90
90
|
files = res['files']
|
91
|
+
|
91
92
|
#upload files
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
93
|
+
blob_ids = Parallel.map(files.keys, self.get_upload_options) do |file|
|
94
|
+
begin
|
95
|
+
Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
|
96
|
+
rescue => e
|
97
|
+
Cnvrg::CLI.log_message("Failed to upload #{file}: #{e.message}", 'red')
|
98
|
+
Cnvrg::Logger.log_error(e)
|
99
|
+
Cnvrg::Logger.log_method(bind: binding)
|
100
|
+
raise e
|
101
|
+
end
|
96
102
|
progress.progress += 1 if progress.present?
|
103
|
+
files[file]["bv_id"]
|
97
104
|
end
|
98
105
|
|
99
106
|
#save files on the server.
|
100
|
-
blob_ids = files.values.map {|f| f['bv_id']}
|
101
107
|
resp = Cnvrg::API.request(@base_resource + "upload_files_save", 'POST', {blob_ids: blob_ids, commit: commit_sha1})
|
102
108
|
unless Cnvrg::CLI.is_response_success(resp, false)
|
103
109
|
raise SignalException.new("Cant save uploaded files to the server.")
|
data/lib/cnvrg/helpers.rb
CHANGED
@@ -33,22 +33,20 @@ module Cnvrg
|
|
33
33
|
end
|
34
34
|
end
|
35
35
|
|
36
|
-
def try_until_success(tries:
|
37
|
-
|
36
|
+
def try_until_success(tries: 3)
|
37
|
+
exception = nil
|
38
|
+
tries.times do |i|
|
38
39
|
begin
|
39
40
|
yield
|
40
41
|
return true
|
41
|
-
rescue Exception => e
|
42
|
-
Cnvrg::Logger.log_info("Error while trying for the #{i} time")
|
43
|
-
Cnvrg::Logger.log_error(e)
|
44
|
-
sleep(5)
|
45
42
|
rescue => e
|
46
43
|
Cnvrg::Logger.log_info("Error while trying for the #{i} time")
|
47
44
|
Cnvrg::Logger.log_error(e)
|
48
|
-
sleep(
|
45
|
+
sleep(1)
|
46
|
+
exception = e
|
49
47
|
end
|
50
48
|
end
|
51
|
-
raise
|
49
|
+
raise exception
|
52
50
|
end
|
53
51
|
|
54
52
|
def get_config
|
@@ -1,5 +1,6 @@
|
|
1
1
|
class Cnvrg::Helpers::Executer
|
2
|
-
def initialize(project: nil, job_type: nil, job_id: nil)
|
2
|
+
def initialize(project: nil, job_type: nil, job_id: nil, image: nil)
|
3
|
+
@image = image
|
3
4
|
@project = project || Cnvrg::Project.new(owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
|
4
5
|
@job_type = job_type || ENV['CNVRG_JOB_TYPE']
|
5
6
|
@job_id = job_id || ENV['CNVRG_JOB_ID']
|
@@ -69,7 +70,7 @@ class Cnvrg::Helpers::Executer
|
|
69
70
|
if command[:type] == "file_exists"
|
70
71
|
puts "Looking for file #{command[:file]}"
|
71
72
|
else
|
72
|
-
puts "Execute #{command[:command]}"
|
73
|
+
puts "Execute #{command[:command]}" unless command[:no_stdout]
|
73
74
|
end
|
74
75
|
execute(command)
|
75
76
|
end
|
@@ -81,6 +82,13 @@ class Cnvrg::Helpers::Executer
|
|
81
82
|
commands.map{|k| k.with_indifferent_access}
|
82
83
|
end
|
83
84
|
|
85
|
+
|
86
|
+
def update_git_commit
|
87
|
+
git_commit = `git rev-parse --verify HEAD`
|
88
|
+
return if git_commit.blank?
|
89
|
+
Cnvrg::API.request("#{base_url}/update_git_commit", "POST", {git_commit: git_commit.strip!})
|
90
|
+
end
|
91
|
+
|
84
92
|
def set_dataset_status(dataset: nil, status: nil)
|
85
93
|
Cnvrg::API.request("#{base_url}/datasets/#{dataset}", "PUT", {status: status})
|
86
94
|
end
|
@@ -131,12 +139,15 @@ class Cnvrg::Helpers::Executer
|
|
131
139
|
output = []
|
132
140
|
start_time = Time.now
|
133
141
|
timeout = cmd[:timeout] || 5*60
|
142
|
+
exit_status = nil
|
134
143
|
t = Thread.new do
|
135
144
|
PTY.spawn(cmd[:command]) do |stdout, stdin, pid, stderr|
|
136
145
|
begin
|
137
|
-
stdout.
|
138
|
-
|
139
|
-
|
146
|
+
if stdout.present?
|
147
|
+
stdout.each do |line|
|
148
|
+
puts line
|
149
|
+
output << {log: line.strip, timestamp: Time.now}
|
150
|
+
end
|
140
151
|
end
|
141
152
|
if stderr.present?
|
142
153
|
stderr.each do |line|
|
@@ -148,16 +159,17 @@ class Cnvrg::Helpers::Executer
|
|
148
159
|
end
|
149
160
|
::Process.wait pid
|
150
161
|
end
|
162
|
+
exit_status = $?.exitstatus
|
151
163
|
end
|
152
164
|
while t.status
|
153
165
|
if Time.now - start_time > timeout
|
154
166
|
puts "Kill thread because of timeout..."
|
155
167
|
errors << {log: "Timeout", timestamp: Time.now}
|
156
168
|
Thread.kill(t)
|
169
|
+
exit_status = -100 ##killed
|
157
170
|
end
|
158
171
|
sleep 1
|
159
172
|
end
|
160
|
-
exit_status = $?.exitstatus
|
161
173
|
end_time = Time.now
|
162
174
|
[exit_status, merge_log_block(output), merge_log_block(errors), start_time, end_time]
|
163
175
|
end
|
@@ -185,7 +197,11 @@ class Cnvrg::Helpers::Executer
|
|
185
197
|
|
186
198
|
|
187
199
|
def job_log(logs, level: 'info', step: nil)
|
188
|
-
|
200
|
+
if @job_type == "image"
|
201
|
+
@image.job_log(logs, level: level, step: step)
|
202
|
+
else
|
203
|
+
@project.job_log(logs, level: level, step: step, job_type: @job_type, job_id: @job_id)
|
204
|
+
end
|
189
205
|
end
|
190
206
|
|
191
207
|
end
|
data/lib/cnvrg/image.rb
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
module Cnvrg
|
2
|
+
class Image
|
3
|
+
# attr_reader :image_name, :image_tag, :is_docker, :project_slug, :commit_id, :owner, :port, :image_slug
|
4
|
+
|
5
|
+
|
6
|
+
def initialize(image_id)
|
7
|
+
begin
|
8
|
+
@cli = Cnvrg::CLI.new
|
9
|
+
home_dir = File.expand_path('~')
|
10
|
+
config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
|
11
|
+
@owner = config.to_h[:owner]
|
12
|
+
@username = config.to_h[:username]
|
13
|
+
@image_id = image_id
|
14
|
+
rescue => e
|
15
|
+
@owner = ""
|
16
|
+
@username = ""
|
17
|
+
@cli.log_message("cnvrg is not configured")
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def build
|
22
|
+
image_data = get_image_data
|
23
|
+
file_name = "Dockerfile-#{@image_id}"
|
24
|
+
File.new(file_name, "w+")
|
25
|
+
if image_data["reqs_file_path"].present? and image_data["from_image_name"]
|
26
|
+
File.open(file_name, "w+") do |i|
|
27
|
+
i.write("FROM #{image_data["from_image_name"]}")
|
28
|
+
i.write("ADD requirements.txt requirements.txt")
|
29
|
+
i.write("RUN pip3 install -r requirements.txt")
|
30
|
+
end
|
31
|
+
else
|
32
|
+
open(file_name, 'wb') do |file|
|
33
|
+
file << open(image_data["docker_file_path"]).read
|
34
|
+
end
|
35
|
+
end
|
36
|
+
command = {:type=>"notify",
|
37
|
+
:title=>"docker build",
|
38
|
+
:logs=>true,
|
39
|
+
:command=>"sudo docker build . -t #{image_data["docker_name"]} -f #{file_name}"}
|
40
|
+
|
41
|
+
@executer = Helpers::Executer.new(project: @project, job_type: "image", job_id: @image_id, image: self)
|
42
|
+
exit_status, output, errors, _, _ = @executer.execute(command)
|
43
|
+
all_logs = join_logs(output, errors)
|
44
|
+
if exit_status > 0
|
45
|
+
raise StandardError.new(all_logs)
|
46
|
+
end
|
47
|
+
if ENV["CNVRG_IMAGE_BUILD_USERNAME"].present? and ENV["CNVRG_IMAGE_BUILD_PASSWORD"].present?
|
48
|
+
command = {:type=>"notify",
|
49
|
+
:no_stdout => true,
|
50
|
+
:title=>"docker login",
|
51
|
+
:logs=>true,
|
52
|
+
:command=>"sudo docker login --username=#{ENV["CNVRG_IMAGE_BUILD_USERNAME"]} --password=\"#{ENV["CNVRG_IMAGE_BUILD_PASSWORD"]}\""}
|
53
|
+
exit_status, output, errors, _, _ = @executer.execute(command)
|
54
|
+
all_logs = join_logs(output, errors)
|
55
|
+
if exit_status > 0
|
56
|
+
raise StandardError.new(all_logs)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
command = {:type=>"notify",
|
60
|
+
:title=>"docker push",
|
61
|
+
:logs=>true,
|
62
|
+
:command=>"sudo docker push #{image_data["docker_name"]}"}
|
63
|
+
exit_status, output, errors, _, _ = @executer.execute(command)
|
64
|
+
all_logs = join_logs(output, errors)
|
65
|
+
if exit_status > 0
|
66
|
+
raise StandardError.new(all_logs)
|
67
|
+
end
|
68
|
+
post_build_update(true)
|
69
|
+
rescue => e
|
70
|
+
post_build_update(false, e.message)
|
71
|
+
end
|
72
|
+
|
73
|
+
def get_image_data
|
74
|
+
response = Cnvrg::API.request("users/#{@owner}/images/#{@image_id}/image_start_build", 'GET')
|
75
|
+
CLI.is_response_success(response)
|
76
|
+
return response["image"]
|
77
|
+
end
|
78
|
+
|
79
|
+
def post_build_update(success, message = "")
|
80
|
+
response = Cnvrg::API.request("users/#{@owner}/images/#{@image_id}/image_end_build", 'POST', {success: success, message: message})
|
81
|
+
CLI.is_response_success(response)
|
82
|
+
return response["image"]
|
83
|
+
end
|
84
|
+
|
85
|
+
|
86
|
+
def job_log(logs, level: 'info', step: nil, job_type: "image", job_id: @image_id)
|
87
|
+
logs = [logs].flatten
|
88
|
+
logs.each_slice(10).each do |temp_logs|
|
89
|
+
Cnvrg::API.request("users/#{@owner}/images/#{@image_id}/log", "POST", {job_type: job_type, job_id: job_id, logs: temp_logs, log_level: level, step: step, timestamp: Time.now})
|
90
|
+
sleep(1)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
|
95
|
+
def join_logs(output, errors)
|
96
|
+
output.map{ |o| o[:logs]}.join(" ") + " " + errors.map{ |o| o[:logs]}.join(" ")
|
97
|
+
end
|
98
|
+
|
99
|
+
end
|
100
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
|
2
|
+
module Cnvrg
|
3
|
+
class ImageCli < SubCommandBase
|
4
|
+
|
5
|
+
desc 'image build --image=IMAGE_ID', 'Build image job', :hide => true
|
6
|
+
method_option :image_id, :type => :string, :aliases => ["--image"]
|
7
|
+
def build_image_job()
|
8
|
+
begin
|
9
|
+
@cli = Cnvrg::CLI.new
|
10
|
+
@cli.verify_logged_in(false)
|
11
|
+
@cli.log_start(__method__, args, options)
|
12
|
+
@cli.log_message("build image started", Thor::Shell::Color::BLUE)
|
13
|
+
image = Image.new(options["image_id"])
|
14
|
+
image.build
|
15
|
+
@cli.log_message("Image build completed successfully", Thor::Shell::Color::BLUE)
|
16
|
+
rescue => e
|
17
|
+
@cli.log_message("Image build completed with an error", Thor::Shell::Color::RED)
|
18
|
+
@cli.log_error(e)
|
19
|
+
exit(1)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
|
24
|
+
end
|
25
|
+
end
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.5.9.
|
4
|
+
version: 1.5.9.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2019-10-
|
13
|
+
date: 2019-10-24 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: bundler
|
@@ -418,6 +418,8 @@ files:
|
|
418
418
|
- lib/cnvrg/helpers.rb
|
419
419
|
- lib/cnvrg/helpers/executer.rb
|
420
420
|
- lib/cnvrg/hyper.rb
|
421
|
+
- lib/cnvrg/image.rb
|
422
|
+
- lib/cnvrg/image_cli.rb
|
421
423
|
- lib/cnvrg/job_cli.rb
|
422
424
|
- lib/cnvrg/logger.rb
|
423
425
|
- lib/cnvrg/org_helpers.rb
|