cnvrg 0.7.7 → 0.7.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/cnvrg.gemspec +1 -7
- data/lib/cnvrg/api.rb +15 -1
- data/lib/cnvrg/cli.rb +286 -722
- data/lib/cnvrg/cli/flow.rb +116 -0
- data/lib/cnvrg/cli/subcommand.rb +26 -0
- data/lib/cnvrg/cli/task.rb +116 -0
- data/lib/cnvrg/data.rb +0 -1
- data/lib/cnvrg/datafiles.rb +55 -35
- data/lib/cnvrg/dataset.rb +50 -9
- data/lib/cnvrg/files.rb +263 -76
- data/lib/cnvrg/flow.rb +75 -0
- data/lib/cnvrg/helpers.rb +54 -0
- data/lib/cnvrg/hyper.rb +21 -0
- data/lib/cnvrg/logger.rb +102 -0
- data/lib/cnvrg/project.rb +86 -19
- data/lib/cnvrg/task.rb +165 -0
- data/lib/cnvrg/version.rb +2 -1
- metadata +9 -2
data/lib/cnvrg/helpers.rb
CHANGED
@@ -19,6 +19,42 @@ module Cnvrg
|
|
19
19
|
end
|
20
20
|
end
|
21
21
|
|
22
|
+
def try_until_success(tries: 5)
|
23
|
+
(0..tries).each do |i|
|
24
|
+
begin
|
25
|
+
yield
|
26
|
+
return true
|
27
|
+
rescue Exception => e
|
28
|
+
Cnvrg::Logger.log_info("Error while trying for the #{i} time")
|
29
|
+
Cnvrg::Logger.log_error(e)
|
30
|
+
sleep(5)
|
31
|
+
rescue => e
|
32
|
+
Cnvrg::Logger.log_info("Error while trying for the #{i} time")
|
33
|
+
Cnvrg::Logger.log_error(e)
|
34
|
+
sleep(5)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
raise StandardError.new("Failed")
|
38
|
+
end
|
39
|
+
|
40
|
+
def get_config
|
41
|
+
home_dir = File.expand_path('~')
|
42
|
+
config = {}
|
43
|
+
begin
|
44
|
+
if File.exist? home_dir+"/.cnvrg/config.yml"
|
45
|
+
config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
|
46
|
+
end
|
47
|
+
end
|
48
|
+
return config
|
49
|
+
end
|
50
|
+
|
51
|
+
def set_config(config)
|
52
|
+
home_dir = File.expand_path('~')
|
53
|
+
File.open("#{home_dir}/.cnvrg/config.yml", "w"){|f| f.write config.to_yaml }
|
54
|
+
return config
|
55
|
+
end
|
56
|
+
|
57
|
+
|
22
58
|
def remote_url
|
23
59
|
home_dir = File.expand_path('~')
|
24
60
|
config = ""
|
@@ -38,6 +74,21 @@ module Cnvrg
|
|
38
74
|
return config.to_h[:api].gsub("/api", "")
|
39
75
|
end
|
40
76
|
end
|
77
|
+
|
78
|
+
def server_version
|
79
|
+
config = self.get_config
|
80
|
+
config[:version].try(:to_i) || 0
|
81
|
+
end
|
82
|
+
|
83
|
+
def update_version(version)
|
84
|
+
config = self.get_config
|
85
|
+
if config[:version].to_s.eql? version
|
86
|
+
return
|
87
|
+
end
|
88
|
+
config[:version] = version
|
89
|
+
self.set_config(config)
|
90
|
+
end
|
91
|
+
|
41
92
|
def is_verify_ssl
|
42
93
|
home_dir = File.expand_path('~')
|
43
94
|
config = ""
|
@@ -94,6 +145,7 @@ module Cnvrg
|
|
94
145
|
end
|
95
146
|
|
96
147
|
def cnvrgignore_content
|
148
|
+
#TODO: cnvrg ignore add .conflict
|
97
149
|
%{
|
98
150
|
# cnvrg ignore: Ignore the following directories and files
|
99
151
|
# for example:
|
@@ -101,6 +153,8 @@ module Cnvrg
|
|
101
153
|
# some_file.txt
|
102
154
|
.git*
|
103
155
|
.gitignore
|
156
|
+
*.conflict
|
157
|
+
*.deleted
|
104
158
|
}.strip
|
105
159
|
end
|
106
160
|
|
data/lib/cnvrg/hyper.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
module Cnvrg
|
2
|
+
class Hyper
|
3
|
+
def initialize(project_path, path)
|
4
|
+
@project = Cnvrg::Project.new(project_path)
|
5
|
+
@content = YAML.load_file(path)
|
6
|
+
@base_resource = "users/#{@project.owner}"
|
7
|
+
@params = []
|
8
|
+
end
|
9
|
+
|
10
|
+
def resolve_params
|
11
|
+
resp = Cnvrg::API.request(@base_resource + "/resolve_grid", "POST", {hyper_search: @content})
|
12
|
+
unless Cnvrg::CLI.is_response_success(resp, false)
|
13
|
+
return nil
|
14
|
+
end
|
15
|
+
resp['result']['params'].each do |param|
|
16
|
+
@params << {key: param.first.keys.first, value: param.map{|p| p.values}.flatten.join(',')}
|
17
|
+
end
|
18
|
+
@params
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
data/lib/cnvrg/logger.rb
ADDED
@@ -0,0 +1,102 @@
|
|
1
|
+
module Cnvrg
|
2
|
+
module Logger
|
3
|
+
extend self
|
4
|
+
def log_handler
|
5
|
+
begin
|
6
|
+
date = DateTime.now.strftime("%m_%d_%Y")
|
7
|
+
home_dir = File.expand_path('~')
|
8
|
+
|
9
|
+
if !File.directory? home_dir + "/.cnvrg"
|
10
|
+
FileUtils.mkdir_p([home_dir + "/.cnvrg", home_dir + "/.cnvrg/tmp"])
|
11
|
+
end
|
12
|
+
if !File.exist?(home_dir + "/.cnvrg/config.yml")
|
13
|
+
FileUtils.touch [home_dir + "/.cnvrg/config.yml"]
|
14
|
+
end
|
15
|
+
logfile = File.expand_path('~') + "/.cnvrg/log_#{date}.log"
|
16
|
+
if !File.exist? logfile
|
17
|
+
FileUtils.touch([logfile])
|
18
|
+
yesterday = get_start_day - 86399
|
19
|
+
date = yesterday.strftime("%m_%d_%Y")
|
20
|
+
|
21
|
+
logfile_old = File.expand_path('~') + "/.cnvrg/log_#{date}.log"
|
22
|
+
count = 0
|
23
|
+
while not File.exist? logfile_old and count < 60
|
24
|
+
yesterday = yesterday - 86399
|
25
|
+
date = yesterday.strftime("%m_%d_%Y")
|
26
|
+
logfile_old = File.expand_path('~') + "/.cnvrg/log_#{date}.log"
|
27
|
+
count += 1
|
28
|
+
end
|
29
|
+
if File.exist? logfile_old
|
30
|
+
@files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
|
31
|
+
@files.upload_log_file(logfile_old, "log_#{date}.log", yesterday)
|
32
|
+
FileUtils.remove logfile_old
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
config = LogStashLogger.configure do |config|
|
37
|
+
config.customize_event do |event|
|
38
|
+
event.remove('@version')
|
39
|
+
event.remove('severity')
|
40
|
+
end
|
41
|
+
end
|
42
|
+
$log = LogStashLogger.new(type: :file, path: logfile, sync: true, config: config)
|
43
|
+
self.remove_old_log_files
|
44
|
+
rescue
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def remove_old_log_files()
|
49
|
+
begin
|
50
|
+
last_week = (Time.now - (7 * 24 * 60 * 60)).strftime("%Y-%m-%d")
|
51
|
+
home = File.expand_path('~')
|
52
|
+
log_files = Dir["#{home}/.cnvrg/tmp/*.log"]
|
53
|
+
log_files.each do |l|
|
54
|
+
if File.mtime(l).strftime("%Y-%m-%d") < last_week
|
55
|
+
FileUtils.rm_rf(l)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def info(msg)
|
62
|
+
self.log_info(msg)
|
63
|
+
end
|
64
|
+
|
65
|
+
def log_method(bind: nil)
|
66
|
+
return if bind.blank?
|
67
|
+
Cnvrg::Logger.log_handler if $log.blank?
|
68
|
+
arg = {}
|
69
|
+
bind.local_variables.map do |name|
|
70
|
+
arg[name] = bind.local_variable_get(name).try(:to_s)
|
71
|
+
end
|
72
|
+
method = bind.eval('__method__')
|
73
|
+
$log.info method: method, args: arg
|
74
|
+
end
|
75
|
+
|
76
|
+
def log_error(e)
|
77
|
+
Cnvrg::Logger.log_handler if $log.blank?
|
78
|
+
bc = ActiveSupport::BacktraceCleaner.new
|
79
|
+
bc.add_silencer{|line| line =~ /thor/}
|
80
|
+
$log.error message: "An exception #{e.class} was logged during running", type: "error"
|
81
|
+
backtrace = bc.clean(e.backtrace).slice(0,6).map.with_index{|backtrace,idx| "(#{idx}) - #{backtrace}"}.join("; ")
|
82
|
+
$log.error message: e.message, type: "error", backtrace: backtrace
|
83
|
+
end
|
84
|
+
|
85
|
+
def log_error_message(msg)
|
86
|
+
Cnvrg::Logger.log_handler if $log.blank?
|
87
|
+
$log.error message: msg, type: "error"
|
88
|
+
end
|
89
|
+
|
90
|
+
def log_info(msg)
|
91
|
+
Cnvrg::Logger.log_handler if $log.blank?
|
92
|
+
$log.info message: msg, type: "info"
|
93
|
+
end
|
94
|
+
|
95
|
+
def jsonify_message(msg: '', success: true)
|
96
|
+
puts JSON[{
|
97
|
+
"msg": msg,
|
98
|
+
"success": success.to_s
|
99
|
+
}]
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
data/lib/cnvrg/project.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
module Cnvrg
|
3
3
|
class Project
|
4
|
-
attr_reader :slug, :owner, :title, :local_path, :working_dir, :is_git, :is_branch
|
4
|
+
attr_reader :slug, :owner, :title, :local_path, :working_dir, :is_git, :is_branch, :machines
|
5
5
|
|
6
6
|
RemoteURL ||= "https://cnvrg.io"
|
7
7
|
IDXParallelThreads ||= 15
|
@@ -17,10 +17,16 @@ module Cnvrg
|
|
17
17
|
@owner = config[:owner]
|
18
18
|
@is_branch = config[:is_branch]
|
19
19
|
@is_git = config[:git] || false
|
20
|
+
@base_resource = "users/#{@owner}/projects/#{@slug}/"
|
21
|
+
@machines = nil
|
20
22
|
rescue => e
|
21
23
|
end
|
22
24
|
end
|
23
25
|
|
26
|
+
def base_resource
|
27
|
+
"users/#{@owner}/projects/#{@slug}/"
|
28
|
+
end
|
29
|
+
|
24
30
|
|
25
31
|
def last_local_commit
|
26
32
|
idx = YAML.load_file(@local_path + "/.cnvrg/idx.yml")
|
@@ -164,7 +170,7 @@ module Cnvrg
|
|
164
170
|
File.open(project_name + "/.cnvrgignore", "w+") { |f| f.write cnvrgignore }
|
165
171
|
File.open(project_name + "/README.md", "w+") { |f| f.write cnvrgreadme }
|
166
172
|
File.open(project_name + "/src/hyper.yaml", "w+") { |f| f.write cnvrghyper }
|
167
|
-
|
173
|
+
|
168
174
|
rescue
|
169
175
|
return false
|
170
176
|
end
|
@@ -343,8 +349,7 @@ module Cnvrg
|
|
343
349
|
if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
|
344
350
|
old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
345
351
|
else
|
346
|
-
|
347
|
-
old_idx = nil
|
352
|
+
old_idx = {:tree => {}, :commit => nil}
|
348
353
|
end
|
349
354
|
|
350
355
|
tree_idx = Hash.new(0)
|
@@ -376,15 +381,25 @@ module Cnvrg
|
|
376
381
|
end
|
377
382
|
end
|
378
383
|
|
379
|
-
|
384
|
+
old_idx[:tree] = tree_idx
|
380
385
|
|
381
|
-
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write
|
386
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write old_idx.to_yaml }
|
382
387
|
return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
383
388
|
end
|
384
389
|
def get_idx
|
390
|
+
unless File.exists? "#{self.local_path}/.cnvrg/idx.yml"
|
391
|
+
empty_idx = {:commit => nil, :tree => {}}
|
392
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write empty_idx.to_yaml }
|
393
|
+
return empty_idx
|
394
|
+
end
|
385
395
|
YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
386
396
|
end
|
387
397
|
|
398
|
+
def set_idx(idx)
|
399
|
+
FileUtils.mkdir_p("#{self.local_path}/.cnvrg")
|
400
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
|
401
|
+
end
|
402
|
+
|
388
403
|
def clone(remote=0, commit)
|
389
404
|
response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/clone", 'POST', {project_slug: self.slug, remote: remote, commit: commit})
|
390
405
|
return response
|
@@ -395,34 +410,70 @@ module Cnvrg
|
|
395
410
|
return response
|
396
411
|
end
|
397
412
|
|
398
|
-
def compare_idx(new_branch,
|
399
|
-
|
400
|
-
|
413
|
+
def compare_idx(new_branch, force:false, deploy: false, in_exp:false, specific_files: [], download: false)
|
414
|
+
is_download = download
|
415
|
+
if is_download
|
416
|
+
local_idx = self.get_idx
|
401
417
|
else
|
418
|
+
#upload
|
402
419
|
local_idx = self.generate_idx(deploy: deploy)
|
403
420
|
end
|
421
|
+
commit = local_idx[:commit]
|
422
|
+
tree = local_idx[:tree]
|
404
423
|
ignore_list = self.send_ignore_list()
|
405
|
-
if force or
|
424
|
+
if force or specific_files.present?
|
406
425
|
added = []
|
407
|
-
if
|
426
|
+
if specific_files.present?
|
408
427
|
added = specific_files
|
409
|
-
elsif
|
428
|
+
elsif tree.present?
|
410
429
|
added << local_idx[:tree].keys
|
411
430
|
added.flatten!
|
412
431
|
end
|
413
432
|
response ={"result"=> {"commit"=>nil,"tree"=> {"added"=> added,
|
414
433
|
"updated_on_server"=> [],
|
415
434
|
"updated_on_local"=> [],
|
435
|
+
"update_local" => [],
|
416
436
|
"deleted"=> [],
|
417
437
|
"conflicts"=> []} } }
|
418
438
|
return response
|
419
|
-
|
420
439
|
end
|
421
|
-
|
422
|
-
|
440
|
+
#we dont want to send it on download - we only compare between commits sha1 in download.
|
441
|
+
if is_download
|
442
|
+
#the new server doesnt need the tree, but the old probably needs :X
|
443
|
+
local_idx[:tree] = {} if Cnvrg::Helpers.server_version > 0
|
444
|
+
end
|
445
|
+
response = Cnvrg::API.request(@base_resource + "status", 'POST', {idx: local_idx, new_branch: new_branch,
|
446
|
+
current_commit: commit,ignore:ignore_list, force:force,in_exp:in_exp, download: download})
|
447
|
+
|
423
448
|
CLI.is_response_success(response,true)
|
449
|
+
if is_download
|
450
|
+
if Cnvrg::Helpers.server_version > 0
|
451
|
+
#trying to optimize the download using resolver
|
452
|
+
resolve = response['result']['tree']['resolver'] || tree #tree of file -> sha1 from current commit to check conflicts
|
453
|
+
destination_files = response['result']['tree']['destination'] || {} #tree of file -> sha1 from last commit to check files that already downloaded
|
454
|
+
else
|
455
|
+
resolve = tree
|
456
|
+
destination_files = {}
|
457
|
+
end
|
458
|
+
@files = self.get_files
|
459
|
+
local_tree = @files.calculate_sha1(resolve.keys)
|
460
|
+
changed_files = resolve.keys.select{|file| local_tree[file] != resolve[file]}
|
461
|
+
|
462
|
+
# means that the user changed the file locally
|
463
|
+
response['result']['tree']['update_local'] = changed_files
|
464
|
+
|
465
|
+
# means that we already downloaded this file and we dont need it anymore
|
466
|
+
downloaded_files = destination_files.keys.select{|file| local_tree[file] == destination_files[file]}
|
467
|
+
response['result']['tree']['added'] -= downloaded_files
|
468
|
+
response['result']['tree']['updated_on_server'] -= downloaded_files
|
469
|
+
end
|
424
470
|
return response
|
425
471
|
end
|
472
|
+
|
473
|
+
def get_files
|
474
|
+
Cnvrg::Files.new(self.owner, self.slug, project_home: @local_path)
|
475
|
+
end
|
476
|
+
|
426
477
|
def jump_idx(destination: self.last_local_commit)
|
427
478
|
local_idx = self.generate_idx
|
428
479
|
ignore_list = self.send_ignore_list
|
@@ -497,7 +548,7 @@ module Cnvrg
|
|
497
548
|
return response
|
498
549
|
end
|
499
550
|
def update_idx_with_commit!(commit)
|
500
|
-
idx_hash =
|
551
|
+
idx_hash = self.get_idx
|
501
552
|
idx_hash[:commit] = commit
|
502
553
|
|
503
554
|
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
|
@@ -506,11 +557,27 @@ module Cnvrg
|
|
506
557
|
|
507
558
|
def revert(working_dir)
|
508
559
|
FileUtils.rm_rf working_dir
|
509
|
-
|
510
|
-
# CLI.is_response_success(response)
|
560
|
+
end
|
511
561
|
|
562
|
+
def init_machines
|
563
|
+
Cnvrg::Logger.log_info("Init machines.")
|
564
|
+
resp = Cnvrg::API.request("users/#{@owner}/machines", "GET")
|
565
|
+
return unless Cnvrg::CLI.is_response_success(resp, false)
|
566
|
+
@machines = resp['result']['machines']
|
567
|
+
end
|
512
568
|
|
513
|
-
|
569
|
+
def get_machines
|
570
|
+
init_machines if @machines.nil?
|
571
|
+
@machines = false if @machines.nil?
|
572
|
+
@machines || []
|
573
|
+
end
|
574
|
+
|
575
|
+
def check_machine(machine)
|
576
|
+
Cnvrg::Logger.log_info("Check if #{machine} machine exists")
|
577
|
+
machines = get_machines
|
578
|
+
return true if machines.blank?
|
579
|
+
machines.include? machine
|
580
|
+
end
|
514
581
|
|
515
582
|
end
|
516
583
|
end
|
data/lib/cnvrg/task.rb
ADDED
@@ -0,0 +1,165 @@
|
|
1
|
+
module Cnvrg
|
2
|
+
class Task
|
3
|
+
attr_accessor :title, :path, :source
|
4
|
+
def initialize(project_path, path: nil, content: {})
|
5
|
+
@path = path
|
6
|
+
@content = content
|
7
|
+
@source = 'flow' if @content.present?
|
8
|
+
@source = 'file' if @path.present?
|
9
|
+
@project = Cnvrg::Project.new(project_path)
|
10
|
+
@project_path = project_path
|
11
|
+
#generic
|
12
|
+
@type = nil #type can be exec, data, library, deploy
|
13
|
+
@compute = 'medium'
|
14
|
+
@title = nil
|
15
|
+
@uid = nil
|
16
|
+
@params = {}
|
17
|
+
|
18
|
+
#exec
|
19
|
+
@cmd = nil
|
20
|
+
@params_path = nil
|
21
|
+
|
22
|
+
#library
|
23
|
+
@library = nil
|
24
|
+
|
25
|
+
#dataset
|
26
|
+
@dataset = nil
|
27
|
+
@query = nil
|
28
|
+
|
29
|
+
#deploy
|
30
|
+
@function = nil
|
31
|
+
|
32
|
+
|
33
|
+
@base_resource = @project.base_resource + "/tasks"
|
34
|
+
|
35
|
+
self.reload_task
|
36
|
+
end
|
37
|
+
|
38
|
+
def save
|
39
|
+
path = @path || gen_path
|
40
|
+
File.open(path, 'w'){|f| f.write(get_content.to_yaml)}
|
41
|
+
end
|
42
|
+
|
43
|
+
def reload_task
|
44
|
+
task_raw = get_content
|
45
|
+
@title = task_raw[:title]
|
46
|
+
@type = task_raw[:type]
|
47
|
+
@uid = task_raw[:uid]
|
48
|
+
@title = task_raw[:title] || @uid
|
49
|
+
@compute = task_raw[:compute] || @compute
|
50
|
+
|
51
|
+
case @type
|
52
|
+
when 'exec'
|
53
|
+
@cmd = task_raw[:cmd]
|
54
|
+
init_params(task_raw)
|
55
|
+
when 'library'
|
56
|
+
@library = task_raw[:library]
|
57
|
+
init_params(task_raw)
|
58
|
+
when 'data'
|
59
|
+
@dataset = task_raw[:dataset]
|
60
|
+
@query = task_raw[:query]
|
61
|
+
when 'deploy'
|
62
|
+
@cmd = task_raw[:cmd]
|
63
|
+
@function = task_raw[:function]
|
64
|
+
else
|
65
|
+
error("Cant parse task of type #{@type}")
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def verify_task
|
70
|
+
case @type
|
71
|
+
when 'exec'
|
72
|
+
verify_exec
|
73
|
+
when 'data'
|
74
|
+
verify_data
|
75
|
+
when 'deploy'
|
76
|
+
verify_deploy
|
77
|
+
when 'library'
|
78
|
+
verify_library
|
79
|
+
else
|
80
|
+
error("Cant parse task of type #{@type}")
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
|
85
|
+
def to_api
|
86
|
+
get_content.merge(params: @params)
|
87
|
+
end
|
88
|
+
|
89
|
+
def run
|
90
|
+
verify_task
|
91
|
+
if @type == 'data'
|
92
|
+
raise StandardError.new("Data Tasks are not runnable")
|
93
|
+
end
|
94
|
+
resp = Cnvrg::API.request(@base_resource, "POST", {task: to_api})
|
95
|
+
Cnvrg::CLI.is_response_success(resp, true)
|
96
|
+
Cnvrg::Helpers.remote_url + resp['result']['url']
|
97
|
+
end
|
98
|
+
|
99
|
+
private
|
100
|
+
def verify_compute
|
101
|
+
unless @project.check_machine(@compute)
|
102
|
+
raise StandardError.new("Cant find #{@compute} machine in project.")
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
def init_params(task_raw)
|
107
|
+
@params_path = task_raw[:params_path].presence
|
108
|
+
@params = task_raw[:params] || @params
|
109
|
+
if @params_path.present?
|
110
|
+
@hyper = Cnvrg::Hyper.new(@project_path, @params_path)
|
111
|
+
@params = @hyper.resolve_params
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def verify_exec
|
116
|
+
if @cmd.blank?
|
117
|
+
error("Cant find command")
|
118
|
+
end
|
119
|
+
verify_compute
|
120
|
+
end
|
121
|
+
|
122
|
+
def verify_data
|
123
|
+
if @dataset.blank?
|
124
|
+
error("Cant find dataset slug")
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def verify_deploy
|
129
|
+
error("Cant find command") if @cmd.blank?
|
130
|
+
error("Cant find function") if @function.blank?
|
131
|
+
verify_compute
|
132
|
+
end
|
133
|
+
|
134
|
+
def verify_library
|
135
|
+
error("Cant find library") if @library.blank?
|
136
|
+
end
|
137
|
+
|
138
|
+
def get_content
|
139
|
+
return @content if @source == 'flow'
|
140
|
+
unless File.exists? @path
|
141
|
+
raise StandardError.new("Cant find task in #{@path}")
|
142
|
+
end
|
143
|
+
YAML.load_file(@path)
|
144
|
+
end
|
145
|
+
|
146
|
+
def gen_path
|
147
|
+
@title ||= "#{@type.capitalize}Task"
|
148
|
+
unless File.exists? @title
|
149
|
+
@path = "#{@title}.task.yaml"
|
150
|
+
return @path
|
151
|
+
end
|
152
|
+
i = 0
|
153
|
+
while File.exists? "#{@title}_#{i}.task.yaml"
|
154
|
+
i += 1
|
155
|
+
end
|
156
|
+
@path = "#{@title}.task.yaml"
|
157
|
+
return @path
|
158
|
+
end
|
159
|
+
|
160
|
+
def error(msg)
|
161
|
+
raise StandardError.new("task: #{@uid} - #{msg}")
|
162
|
+
end
|
163
|
+
|
164
|
+
end
|
165
|
+
end
|