cnvrg 1.9.9.9.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/bin/cnvrg +9 -0
- data/cnvrg.gemspec +47 -0
- data/lib/cnvrg.rb +7 -0
- data/lib/cnvrg/Images.rb +351 -0
- data/lib/cnvrg/api.rb +247 -0
- data/lib/cnvrg/api_v2.rb +14 -0
- data/lib/cnvrg/auth.rb +79 -0
- data/lib/cnvrg/cli.rb +5715 -0
- data/lib/cnvrg/cli/flow.rb +166 -0
- data/lib/cnvrg/cli/library_cli.rb +33 -0
- data/lib/cnvrg/cli/subcommand.rb +28 -0
- data/lib/cnvrg/cli/task.rb +116 -0
- data/lib/cnvrg/colors.rb +8 -0
- data/lib/cnvrg/connect_job_ssh.rb +31 -0
- data/lib/cnvrg/data.rb +335 -0
- data/lib/cnvrg/datafiles.rb +1325 -0
- data/lib/cnvrg/dataset.rb +892 -0
- data/lib/cnvrg/downloader/client.rb +101 -0
- data/lib/cnvrg/downloader/clients/azure_client.rb +45 -0
- data/lib/cnvrg/downloader/clients/gcp_client.rb +50 -0
- data/lib/cnvrg/downloader/clients/s3_client.rb +78 -0
- data/lib/cnvrg/experiment.rb +209 -0
- data/lib/cnvrg/files.rb +1047 -0
- data/lib/cnvrg/flow.rb +137 -0
- data/lib/cnvrg/helpers.rb +422 -0
- data/lib/cnvrg/helpers/agent.rb +188 -0
- data/lib/cnvrg/helpers/executer.rb +213 -0
- data/lib/cnvrg/hyper.rb +21 -0
- data/lib/cnvrg/image.rb +113 -0
- data/lib/cnvrg/image_cli.rb +25 -0
- data/lib/cnvrg/job_cli.rb +73 -0
- data/lib/cnvrg/job_ssh.rb +48 -0
- data/lib/cnvrg/logger.rb +111 -0
- data/lib/cnvrg/org_helpers.rb +5 -0
- data/lib/cnvrg/project.rb +822 -0
- data/lib/cnvrg/result.rb +29 -0
- data/lib/cnvrg/runner.rb +49 -0
- data/lib/cnvrg/ssh.rb +94 -0
- data/lib/cnvrg/storage.rb +128 -0
- data/lib/cnvrg/task.rb +165 -0
- data/lib/cnvrg/version.rb +3 -0
- metadata +460 -0
@@ -0,0 +1,166 @@
|
|
1
|
+
require 'cnvrg/cli'
|
2
|
+
require 'cnvrg/data'
|
3
|
+
require 'cnvrg/task'
|
4
|
+
require 'cnvrg/flow'
|
5
|
+
require 'cnvrg/hyper'
|
6
|
+
require 'cnvrg/cli/subcommand'
|
7
|
+
require 'cnvrg/cli/task'
|
8
|
+
require 'thor'
|
9
|
+
module Cnvrg
|
10
|
+
module Commands
|
11
|
+
class Flow < SubCommand
|
12
|
+
|
13
|
+
def initialize(*args)
|
14
|
+
super
|
15
|
+
unless @curr_dir.present?
|
16
|
+
@cli.log_message("Not On Project Dir, exiting", Thor::Color::RED)
|
17
|
+
exit(1)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
desc "flow import --file='flow.yaml'", "Import flow to the web"
|
22
|
+
method_option :file, :type => :string, :aliases => ["-f", "--file"]
|
23
|
+
def import()
|
24
|
+
run_flow_internal(file: options[:file], run: false)
|
25
|
+
end
|
26
|
+
|
27
|
+
desc "flow run --file='flow.yaml'", "Import flow to the web"
|
28
|
+
method_option :file, :type => :string, :aliases => ["-f", "--file"]
|
29
|
+
def run()
|
30
|
+
run_flow_internal(file: options[:file], run: true)
|
31
|
+
end
|
32
|
+
|
33
|
+
no_commands {
|
34
|
+
def run_flow_internal(file: nil, run: false)
|
35
|
+
if options[:file].blank?
|
36
|
+
Cnvrg::CLI.log_message("Flow title is required for export. please use --title='Flow 1'")
|
37
|
+
return
|
38
|
+
end
|
39
|
+
if not File.exists? options[:file]
|
40
|
+
Cnvrg::CLI.log_message("Cant find file in path #{options[:file]}.")
|
41
|
+
return
|
42
|
+
end
|
43
|
+
payload = YAML.safe_load(File.open(file).read)
|
44
|
+
@project = Cnvrg::Project.new(Cnvrg::CLI.get_project_home)
|
45
|
+
begin
|
46
|
+
@flow, version_slug = Flows.create_flow(@project, payload, run: run)
|
47
|
+
rescue => e
|
48
|
+
Cnvrg::CLI.log_message("Error while validating flow: #{e.message}", 'error')
|
49
|
+
return
|
50
|
+
end
|
51
|
+
Cnvrg::CLI.log_message("New flow version was created successfuly!", 'green')
|
52
|
+
Cnvrg::CLI.log_message("you can find the flow in #{@flow.edit_version_href(version_slug)}", 'green')
|
53
|
+
rescue => e
|
54
|
+
Cnvrg::CLI.log_message("Failed while creating flow")
|
55
|
+
Cnvrg::Logger.log_error(e)
|
56
|
+
end
|
57
|
+
}
|
58
|
+
|
59
|
+
desc "flow export --file='flow.yaml' --title='Flow 1' --version='version 1'", "Export flow to the web"
|
60
|
+
method_option :file, :type => :string, :aliases => ["-f", "--file"]
|
61
|
+
method_option :title, :type => :string, :aliases => ["-t", "--title"]
|
62
|
+
method_option :version, :type => :string, :aliases => ["-v", "--version"], :default => "latest"
|
63
|
+
def export()
|
64
|
+
if options[:title].blank?
|
65
|
+
Cnvrg::CLI.log_message("Flow title is required for export. please use --title='Flow 1'", "red")
|
66
|
+
return
|
67
|
+
end
|
68
|
+
|
69
|
+
@project = Cnvrg::Project.new(Cnvrg::CLI.get_project_home)
|
70
|
+
@flow = Flows.new(options[:title], project: @project)
|
71
|
+
filename = @flow.export(options[:version], file: options[:file])
|
72
|
+
Cnvrg::CLI.log_message("Flow was saved successfuly to: #{filename}", 'green')
|
73
|
+
rescue => e
|
74
|
+
Cnvrg::CLI.log_message(e.message, 'red')
|
75
|
+
Cnvrg::Logger.log_error(e)
|
76
|
+
end
|
77
|
+
|
78
|
+
# desc "task", "Running Flow tasks", :hide => true
|
79
|
+
# subcommand 'task', Cnvrg::Commands::Task
|
80
|
+
#
|
81
|
+
# desc "flow verify", "verify that the flow is well formatted"
|
82
|
+
#
|
83
|
+
# def verify(path)
|
84
|
+
# unless @curr_dir.present?
|
85
|
+
# @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
|
86
|
+
# return false
|
87
|
+
# end
|
88
|
+
# @flow = Cnvrg::Flows.new(@curr_dir, path)
|
89
|
+
# @cli.log_message("The Flow is Valid", Thor::Color::GREEN)
|
90
|
+
# end
|
91
|
+
#
|
92
|
+
#
|
93
|
+
# desc "flow run", "run a flow file"
|
94
|
+
# def run(flow_slug)
|
95
|
+
# unless @curr_dir.present?
|
96
|
+
# @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
|
97
|
+
# return false
|
98
|
+
# end
|
99
|
+
# @flow = Cnvrg::Flows.new(flow_slug)
|
100
|
+
# resp = @flow.run
|
101
|
+
# flow_version_href = resp["flow_version"]["href"]
|
102
|
+
# Cnvrg::CLI.log_message("Flow Live results: #{flow_version_href}")
|
103
|
+
# true
|
104
|
+
# end
|
105
|
+
#
|
106
|
+
# desc "flow create", "create a flow file"
|
107
|
+
#
|
108
|
+
# def create
|
109
|
+
# title = ask("Flow Title: ")
|
110
|
+
# title = title.presence || "cnvrg-flow"
|
111
|
+
# relations = []
|
112
|
+
# task = ask_for_relation
|
113
|
+
# while task.compact.size == 2
|
114
|
+
# from, to = task
|
115
|
+
# relations << {from: from, to: to}
|
116
|
+
# task = ask_for_relation
|
117
|
+
# end
|
118
|
+
# start_commit = ask("Flow Starting Commit (empty will set this value to 'latest')")
|
119
|
+
# Cnvrg::Flows.create_flow("#{title}.flow.yaml", {title: title, relations: relations, start_commit: start_commit})
|
120
|
+
# end
|
121
|
+
#
|
122
|
+
#
|
123
|
+
# # desc "flow resolve", "Resolve flow parameters"
|
124
|
+
# # def resolve(path)
|
125
|
+
# # @hyper = Cnvrg::Hyper.new(@curr_dir, path)
|
126
|
+
# # @hyper.resolve_params
|
127
|
+
# # end
|
128
|
+
# #
|
129
|
+
#
|
130
|
+
# private
|
131
|
+
#
|
132
|
+
# def init_tasks
|
133
|
+
# @tasks = Dir.glob("**/*.task*")
|
134
|
+
# end
|
135
|
+
#
|
136
|
+
# def ask_for_relation
|
137
|
+
# init_tasks if @tasks.blank?
|
138
|
+
# to = nil
|
139
|
+
# from = ask_for_task("Task To Start From: [#{@tasks.join(', ')}]")
|
140
|
+
# to = ask_for_task("Task To Go To: [#{@tasks.join(', ')}]") if from.present?
|
141
|
+
# [from, to]
|
142
|
+
# end
|
143
|
+
#
|
144
|
+
# def ask_for_task(text)
|
145
|
+
# verified = false
|
146
|
+
# task = nil
|
147
|
+
# while !verified
|
148
|
+
# task = ask(text)
|
149
|
+
# if task.blank?
|
150
|
+
# return nil
|
151
|
+
# end
|
152
|
+
# begin
|
153
|
+
# Cnvrg::Task.new(@curr_dir, path: task).verify_task
|
154
|
+
# verified = true
|
155
|
+
# rescue => e
|
156
|
+
# end
|
157
|
+
# end
|
158
|
+
# task
|
159
|
+
# end
|
160
|
+
#
|
161
|
+
# def get_all_tasks
|
162
|
+
# @tasks = Dir.glob("*/**.task.yaml")
|
163
|
+
# end
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Cnvrg
|
2
|
+
class LibraryCli < SubCommandBase
|
3
|
+
|
4
|
+
desc "library import", ''
|
5
|
+
def import
|
6
|
+
unless File.exists? "library.yml"
|
7
|
+
Cnvrg::CLI.log_message("Can't find library.yml", 'red')
|
8
|
+
exit(1)
|
9
|
+
end
|
10
|
+
library = YAML.safe_load(File.open("library.yml").read)
|
11
|
+
Cnvrg::CLI.log_message("Archiving library #{library["title"]}")
|
12
|
+
files = Dir["**/*"].select{|file| not File.directory?(file)}
|
13
|
+
File.open("archive.tar.gz", "wb") do |file|
|
14
|
+
Zlib::GzipWriter.wrap(file) do |gzip|
|
15
|
+
Gem::Package::TarWriter.new(gzip) do |tar|
|
16
|
+
files.each do |filename|
|
17
|
+
f = File.open(filename)
|
18
|
+
tar.add_file_simple(filename, 0644, f.size) do |io|
|
19
|
+
io.write(f.read)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
response = Cnvrg::API.request(['users', Cnvrg::CLI.get_owner, 'libraries'].join("/"), "POST_FILE", {relative_path: "archive.tar.gz"})
|
26
|
+
if response["status"] != 200
|
27
|
+
Cnvrg::CLI.log_message("Can't create library: #{response["message"]}")
|
28
|
+
exit(1)
|
29
|
+
end
|
30
|
+
Cnvrg::CLI.log_message("Library Created successfuly", "green")
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
class SubCommand < Thor
|
2
|
+
@cli = nil
|
3
|
+
@curr_dir = nil
|
4
|
+
|
5
|
+
def initialize(*args)
|
6
|
+
super
|
7
|
+
@cli = Cnvrg::CLI.new
|
8
|
+
@curr_dir = @cli.is_cnvrg_dir
|
9
|
+
@config = YAML.load_file(@curr_dir + "/.cnvrg/config.yml") if @curr_dir
|
10
|
+
@helpers = Cnvrg::OrgHelpers.new();
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.banner(command, namespace = nil, subcommand = false)
|
14
|
+
"#{basename} #{command.usage}"
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.subcommand_prefix
|
18
|
+
self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) {|match| match[0].downcase}.gsub(%r{[A-Z]}) {|match| "-#{match[0].downcase}"}
|
19
|
+
end
|
20
|
+
|
21
|
+
class << self
|
22
|
+
# Hackery.Take the run method away from Thor so that we can redefine it.
|
23
|
+
def is_thor_reserved_word?(word, type)
|
24
|
+
return false if word == "run"
|
25
|
+
super
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,116 @@
|
|
1
|
+
module Cnvrg
|
2
|
+
module Commands
|
3
|
+
class Task < SubCommand
|
4
|
+
|
5
|
+
def initialize(*args)
|
6
|
+
super
|
7
|
+
unless @curr_dir.present?
|
8
|
+
@cli.log_message("Not On Project Dir, exiting", Thor::Color::RED)
|
9
|
+
exit(1)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
desc "flow task verify", "verify that the task is well formatted"
|
14
|
+
def verify(path)
|
15
|
+
unless @curr_dir.present?
|
16
|
+
@cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
|
17
|
+
return false
|
18
|
+
end
|
19
|
+
@task = Cnvrg::Task.new(@curr_dir, path: path)
|
20
|
+
begin
|
21
|
+
@task.verify_task
|
22
|
+
rescue StandardError => e
|
23
|
+
@cli.log_message("An error during parsing task: #{e.message}", Thor::Color::RED)
|
24
|
+
@cli.log_error(e)
|
25
|
+
return false
|
26
|
+
rescue => e
|
27
|
+
@cli.log_error(e)
|
28
|
+
return false
|
29
|
+
end
|
30
|
+
@cli.log_message("The task verified successfuly", Thor::Color::GREEN)
|
31
|
+
return @task
|
32
|
+
end
|
33
|
+
|
34
|
+
|
35
|
+
desc "flow task create", "Creates new task"
|
36
|
+
def create
|
37
|
+
@project = Cnvrg::Project.new(@curr_dir)
|
38
|
+
|
39
|
+
task = {}
|
40
|
+
task[:title] = ask("Task title", Thor::Color::GREEN)
|
41
|
+
task[:type] = ask("Task Type?", Thor::Color::GREEN, :limited_to => ["exec", "data", "library", "deploy"])
|
42
|
+
case task[:type]
|
43
|
+
when "exec"
|
44
|
+
task[:cmd] = ask("Command to run", Thor::Color::GREEN)
|
45
|
+
task[:params], task[:params_path] = ask_for_params
|
46
|
+
task[:machine] = ask_for_machine
|
47
|
+
when "data"
|
48
|
+
task[:dataset] = ask("Dataset slug", Thor::Color::GREEN)
|
49
|
+
task[:query] = ask("Dataset query", Thor::Color::GREEN)
|
50
|
+
when "library"
|
51
|
+
task[:library] = ask("Library Slug", Thor::Color::GREEN)
|
52
|
+
task[:params], task[:params_path] = ask_for_params
|
53
|
+
task[:machine] = ask_for_machine
|
54
|
+
when "deploy"
|
55
|
+
task[:cmd] = ask("Command to run", Thor::Color::GREEN)
|
56
|
+
task[:function] = ask("Function to run", Thor::Color::GREEN)
|
57
|
+
end
|
58
|
+
@task = Cnvrg::Task.new(@curr_dir, content: task)
|
59
|
+
@task.save
|
60
|
+
@cli.log_message("Task #{@task.title} Saved Successfuly", Thor::Color::GREEN)
|
61
|
+
end
|
62
|
+
|
63
|
+
|
64
|
+
desc "flow task run", "Running specific task"
|
65
|
+
def run(path)
|
66
|
+
begin
|
67
|
+
path = "#{path}.task.yaml" unless path.end_with? '.task.yaml'
|
68
|
+
@task = Cnvrg::Task.new(@curr_dir, path: path)
|
69
|
+
url = @task.run
|
70
|
+
@cli.log_message("Task: #{@task.title} is running, you can track its performance on #{url}", Thor::Color::GREEN)
|
71
|
+
rescue => e
|
72
|
+
@cli.log_message(e.message, Thor::Color::RED)
|
73
|
+
@cli.log_error(e)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
private
|
78
|
+
|
79
|
+
def ask_for_params
|
80
|
+
params = []
|
81
|
+
param_key = ask("Param key: (enter for continue)", Thor::Color::GREEN)
|
82
|
+
while param_key.present?
|
83
|
+
param_value = ask("#{param_key} value: (seperated by commas)", Thor::Color::GREEN)
|
84
|
+
params << {key: param_key, value: param_value}
|
85
|
+
param_key = ask("Param key: (enter for continue)", Thor::Color::GREEN)
|
86
|
+
end
|
87
|
+
if params.blank?
|
88
|
+
params_path = ask("Params from file: ")
|
89
|
+
if params_path.present?
|
90
|
+
while params_path.present? and !File.exists? params_path
|
91
|
+
params_path = ask("Cant find file, please write it again.")
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
return params, params_path
|
96
|
+
end
|
97
|
+
|
98
|
+
def ask_for_machine
|
99
|
+
options = {}
|
100
|
+
all_machines = @project.get_machines
|
101
|
+
options[:limited_to] = all_machines if all_machines.present?
|
102
|
+
ask("Kind of machine to execute on?", Thor::Color::GREEN, options).presence || "medium"
|
103
|
+
end
|
104
|
+
|
105
|
+
|
106
|
+
def on_project
|
107
|
+
unless @curr_dir.present?
|
108
|
+
@cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
|
109
|
+
return false
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
data/lib/cnvrg/colors.rb
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
module Cnvrg
|
2
|
+
class ConnectJobSsh
|
3
|
+
def initialize(job_id)
|
4
|
+
home_dir = File.expand_path('~')
|
5
|
+
config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
|
6
|
+
@owner = config.to_h[:owner]
|
7
|
+
@job_id = job_id
|
8
|
+
rescue => e
|
9
|
+
@owner = ""
|
10
|
+
Cnvrg::Logger.log_info("cnvrg is not configured")
|
11
|
+
end
|
12
|
+
|
13
|
+
def start(username, password)
|
14
|
+
Cnvrg::API_V2.request("#{@owner}/job_ssh/#{@job_id}/start" , 'POST', {username: username, password: password})
|
15
|
+
end
|
16
|
+
|
17
|
+
def status()
|
18
|
+
Cnvrg::API_V2.request("#{@owner}/job_ssh/#{@job_id}/status" , 'GET', nil)
|
19
|
+
end
|
20
|
+
|
21
|
+
def run_portforward_command(pod_name, port, kubeconfig, namespace)
|
22
|
+
command = "kubectl"
|
23
|
+
if kubeconfig.present?
|
24
|
+
command = "kubectl --kubeconfig=#{kubeconfig}"
|
25
|
+
end
|
26
|
+
bashCommand = "#{command} -n #{namespace} port-forward #{pod_name} #{port}:22"
|
27
|
+
puts("\nrunning command #{bashCommand}")
|
28
|
+
`#{bashCommand}`
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
data/lib/cnvrg/data.rb
ADDED
@@ -0,0 +1,335 @@
|
|
1
|
+
require 'cnvrg/cli'
|
2
|
+
require 'thor'
|
3
|
+
class SubCommandBase < Thor
|
4
|
+
def self.banner(command, namespace = nil, subcommand = false)
|
5
|
+
"#{basename} #{command.usage}"
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.subcommand_prefix
|
9
|
+
self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) { |match| match[0].downcase }.gsub(%r{[A-Z]}) { |match| "-#{match[0].downcase}" }
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
module Cnvrg
|
14
|
+
class Data < SubCommandBase
|
15
|
+
class_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
|
16
|
+
|
17
|
+
module ConfigValidation
|
18
|
+
SUCCESS = "success"
|
19
|
+
FAILED = "failed"
|
20
|
+
end
|
21
|
+
|
22
|
+
desc "data init", "Set current directory as dataset directory"
|
23
|
+
method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
|
24
|
+
method_option :bucket, :type => :string, :aliases => ["-b", "--bucket"], :default => ""
|
25
|
+
method_option :title, :type => :string, :aliases => ["-t", "--title"], :default => ""
|
26
|
+
|
27
|
+
def init
|
28
|
+
cli = Cnvrg::CLI.new()
|
29
|
+
public = options["public"]
|
30
|
+
bucket = options["bucket"]
|
31
|
+
title = options["title"]
|
32
|
+
cli.init_data(public, bucket: bucket, title: title)
|
33
|
+
end
|
34
|
+
|
35
|
+
desc "data link DATASET_SLUG", "Set current directory as dataset directory"
|
36
|
+
def link(dataset=nil)
|
37
|
+
begin
|
38
|
+
cli = Cnvrg::CLI.new()
|
39
|
+
cli.verify_logged_in(false)
|
40
|
+
cli.log_start(__method__, args, options)
|
41
|
+
|
42
|
+
if dataset.include? "/"
|
43
|
+
## this case is to support DATASET_URL expected example: domain.com/organization_name/datasets/dataset_name
|
44
|
+
url_parts = dataset.split("/")
|
45
|
+
dataset_index = Cnvrg::Helpers.look_for_in_path(dataset, "datasets")
|
46
|
+
dataset_slug = url_parts[dataset_index + 1]
|
47
|
+
owner = url_parts[dataset_index - 1]
|
48
|
+
raise Exception.new("Can't find all dataset information please check the URL") if dataset_slug.blank? or owner.blank?
|
49
|
+
else
|
50
|
+
## this case is to support DATASET_SLUG expected example: dataset_name
|
51
|
+
# in this case it will take the organization from the config file
|
52
|
+
dataset_slug = dataset
|
53
|
+
raise Exception.new("Please enter dataset name or dataset full url") if dataset_slug.blank?
|
54
|
+
owner = CLI.get_owner
|
55
|
+
end
|
56
|
+
|
57
|
+
config_validation = Dataset.validate_config
|
58
|
+
if config_validation[:validation] == Data::ConfigValidation::SUCCESS
|
59
|
+
cli.log_message(config_validation[:message])
|
60
|
+
return
|
61
|
+
else
|
62
|
+
Cnvrg::Logger.log_error_message(config_validation)
|
63
|
+
FileUtils.rmtree(".cnvrg")
|
64
|
+
end
|
65
|
+
|
66
|
+
cli.log_message("Linking dataset: #{dataset_slug} to the current directory", Thor::Shell::Color::BLUE)
|
67
|
+
success = Dataset.link_dataset(owner: owner, slug: dataset_slug)
|
68
|
+
if success
|
69
|
+
cli.log_message("Dataset: #{dataset_slug} linked successfully to the current directory")
|
70
|
+
else
|
71
|
+
cli.log_message("Linking failed\nAborting", Thor::Shell::Color::RED)
|
72
|
+
end
|
73
|
+
rescue => e
|
74
|
+
Cnvrg::Logger.log_error(e)
|
75
|
+
cli.log_message("Aborting\n#{e.message}", Thor::Shell::Color::RED)
|
76
|
+
exit(1)
|
77
|
+
rescue Exception => e
|
78
|
+
Cnvrg::Logger.log_error(e)
|
79
|
+
cli.log_message("Aborting\n#{e.message}", Thor::Shell::Color::RED)
|
80
|
+
exit(1)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
|
85
|
+
desc "data upload", "Upload files from local dataset directory to remote server"
|
86
|
+
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
87
|
+
method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
|
88
|
+
method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
|
89
|
+
method_option :sync, :type => :boolean, :aliases => ["-s","--sync"], :default => false
|
90
|
+
method_option :tags, :type => :boolean, :aliases => ["--tags"], :desc => "upload file tags", :default => false
|
91
|
+
method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :desc => "upload file tags", :default => 1000
|
92
|
+
method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
|
93
|
+
|
94
|
+
def upload
|
95
|
+
cli = Cnvrg::CLI.new()
|
96
|
+
verbose = options["verbose"]
|
97
|
+
sync = options["sync"]
|
98
|
+
force = options["force"]
|
99
|
+
new_branch = options["new_branch"]
|
100
|
+
chunk_size = options["chunk_size"]
|
101
|
+
tags = options["tags"]
|
102
|
+
message = options["message"]
|
103
|
+
cli.upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:message)
|
104
|
+
end
|
105
|
+
desc 'data sync', 'Synchronise local dataset directory with remote server'
|
106
|
+
method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
|
107
|
+
method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
|
108
|
+
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
109
|
+
method_option :commit, :type => :string, :aliases => ["-c"], :desc => "download specified commit", :default => nil
|
110
|
+
method_option :all_files, :type => :boolean, :aliases => ["--all"], :desc => "download specified commit", :default => false
|
111
|
+
method_option :tags, :type => :boolean, :aliases => ["--tags"], :desc => "upload file tags", :default => false
|
112
|
+
method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
|
113
|
+
method_option :chunk_size, :type => :numeric, :aliases => ["--chunk_size"], :desc => "chunk size to communicate with the server", :default => 1000
|
114
|
+
method_option :init, :type => :boolean, :aliases => ["--initial"], :desc => "initial upload of dataset", :default => false
|
115
|
+
method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
|
116
|
+
|
117
|
+
def sync_data_new()
|
118
|
+
cli = Cnvrg::CLI.new()
|
119
|
+
force = options["force"]
|
120
|
+
new_branch = options["new_branch"]
|
121
|
+
verbose = options["verbose"]
|
122
|
+
commit = options["commit"]
|
123
|
+
all_files = options["all_files"]
|
124
|
+
tags = options["tags"]
|
125
|
+
parallel=options["parallel"]
|
126
|
+
chunk_size = options["chunk_size"]
|
127
|
+
init = options["init"]
|
128
|
+
message = options["message"]
|
129
|
+
cli.sync_data_new(new_branch, force, verbose, commit, all_files, tags, parallel, chunk_size, init, message)
|
130
|
+
end
|
131
|
+
|
132
|
+
desc 'data download', 'Download files from remote server'
|
133
|
+
method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits", :default => false
|
134
|
+
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
135
|
+
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
136
|
+
method_option :commit, :type => :string, :aliases => ["-c"], :desc => "download specified commit", :default => nil
|
137
|
+
method_option :all_files, :type => :boolean, :aliases => ["--all"], :desc => "download specified commit", :default => false
|
138
|
+
|
139
|
+
def download()
|
140
|
+
cli = Cnvrg::CLI.new()
|
141
|
+
verbose = options["verbose"]
|
142
|
+
sync = options["sync"]
|
143
|
+
new_branch = options["new_branch"]
|
144
|
+
commit = options["commit"]
|
145
|
+
all_files = options["all_files"]
|
146
|
+
cli.download_data_new(verbose,sync,new_branch, commit,all_files)
|
147
|
+
end
|
148
|
+
desc 'data clone DATASET_URL', 'Clone dataset'
|
149
|
+
method_option :only_tree, :type => :boolean, :aliases => ["-t", "--tree"], :default => false
|
150
|
+
method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => nil
|
151
|
+
method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
|
152
|
+
method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
|
153
|
+
method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
|
154
|
+
method_option :relative, :type => :boolean, :aliases => ["-rel", "--relative"], :default => false
|
155
|
+
method_option :flatten, :type => :boolean, :aliases => ["-f", "--flatten"], :default => false
|
156
|
+
method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
|
157
|
+
def clone(dataset_url)
|
158
|
+
cli = Cnvrg::CLI.new()
|
159
|
+
only_tree =options[:only_tree]
|
160
|
+
commit =options[:commit]
|
161
|
+
query =options[:query]
|
162
|
+
read = options[:read]
|
163
|
+
remote = options[:remote]
|
164
|
+
soft = options[:soft]
|
165
|
+
flatten = options[:flatten]
|
166
|
+
cli.clone_data(
|
167
|
+
dataset_url,
|
168
|
+
only_tree=only_tree,
|
169
|
+
commit=commit,
|
170
|
+
query=query,
|
171
|
+
read=read,
|
172
|
+
remote=remote,
|
173
|
+
flatten: flatten,
|
174
|
+
relative: options[:relative],
|
175
|
+
soft: soft
|
176
|
+
)
|
177
|
+
end
|
178
|
+
|
179
|
+
desc 'data verify DATASETS_TITLES', 'verify datasets', :hide => true
|
180
|
+
method_option :timeout, :type => :numeric, :aliases => ["-t", "--timeout"], :desc => "Time to wait before returning final answer", :default => nil
|
181
|
+
def verify(*dataset_titles)
|
182
|
+
cli = Cnvrg::CLI.new()
|
183
|
+
timeout =options[:timeout]
|
184
|
+
cli.verify_datasets(dataset_titles, timeout)
|
185
|
+
end
|
186
|
+
|
187
|
+
desc 'data scan', 'lookup datasets', :hide => true
|
188
|
+
def scan()
|
189
|
+
cli = Cnvrg::CLI.new()
|
190
|
+
cli.scan_datasets()
|
191
|
+
end
|
192
|
+
|
193
|
+
desc "data block DATASET_TITLES", 'verifying that datasets exists', hide: true
|
194
|
+
def block(*dataset_slugs)
|
195
|
+
not_verified = true
|
196
|
+
while not_verified
|
197
|
+
not_verified = dataset_slugs.select{|slug| not Dataset.verify_dataset(slug)}.present?
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
desc 'data set --url=DATASET_URL', 'Set dataset url to other url'
|
202
|
+
method_option :url, :type => :string, :aliases => ["--url"], :default => ''
|
203
|
+
def set
|
204
|
+
cli = Cnvrg::CLI.new
|
205
|
+
cli.log_start(__method__)
|
206
|
+
cli.log_handler
|
207
|
+
if options['url'].present?
|
208
|
+
cli.set_data_url(options['url'])
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
|
213
|
+
method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
|
214
|
+
method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
|
215
|
+
method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
|
216
|
+
method_option :override, :type => :boolean, :aliases => ["--override"], :default => false
|
217
|
+
method_option :threads, :type => :numeric, :aliases => ["-t","--threads"], :default => 15
|
218
|
+
method_option :chunk_size, :type => :numeric, :aliases => ["-cs","--chunk"], :default => 1000
|
219
|
+
method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
|
220
|
+
|
221
|
+
def put(dataset_url, *files)
|
222
|
+
cli = Cnvrg::CLI.new()
|
223
|
+
dir = options[:dir]
|
224
|
+
force = options[:force]
|
225
|
+
override = options[:override]
|
226
|
+
commit = options[:commit]
|
227
|
+
message = options[:message]
|
228
|
+
threads = options[:threads]
|
229
|
+
chunk_size = options[:chunk_size]
|
230
|
+
cli.data_put(
|
231
|
+
dataset_url,
|
232
|
+
files: files,
|
233
|
+
dir: dir,
|
234
|
+
commit: commit,
|
235
|
+
force: force,
|
236
|
+
override: override,
|
237
|
+
threads: threads,
|
238
|
+
chunk_size: chunk_size,
|
239
|
+
message: message
|
240
|
+
)
|
241
|
+
end
|
242
|
+
|
243
|
+
desc 'data rm DATASET_URL FILES_PREFIX', 'Delete selected files from remote server'
|
244
|
+
method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
|
245
|
+
def rm(dataset_url, *regex_list)
|
246
|
+
cli = Cnvrg::CLI.new()
|
247
|
+
message = options[:message]
|
248
|
+
cli.data_rm(dataset_url, regex_list: regex_list, message: message)
|
249
|
+
end
|
250
|
+
|
251
|
+
desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'
|
252
|
+
method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
|
253
|
+
method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
|
254
|
+
method_option :flatten, :type => :boolean, :aliases => ["-f", "--flatten"], :default => false
|
255
|
+
def clone_query(dataset_url)
|
256
|
+
cli = Cnvrg::CLI.new()
|
257
|
+
query = options[:query]
|
258
|
+
flatten = options[:flatten]
|
259
|
+
soft =options[:soft]
|
260
|
+
cli.clone_data_query(dataset_url,query=query, flatten, soft: soft)
|
261
|
+
end
|
262
|
+
|
263
|
+
desc 'data delete DATASET_SLUG', 'Delete dataset'
|
264
|
+
def delete(dataset_slug)
|
265
|
+
cli = Cnvrg::CLI.new()
|
266
|
+
cli.delete_data(dataset_slug)
|
267
|
+
|
268
|
+
end
|
269
|
+
desc 'data list', 'Show list of all datasets'
|
270
|
+
def list()
|
271
|
+
cli = Cnvrg::CLI.new()
|
272
|
+
|
273
|
+
cli.list_dataset()
|
274
|
+
|
275
|
+
end
|
276
|
+
|
277
|
+
desc 'data commits URL/SLUG', 'List all commits for a given dataset'
|
278
|
+
method_option :commit_sha1, :type => :string, :aliases => ["-c", "--commit"], :default => nil
|
279
|
+
def commits(dataset_url)
|
280
|
+
cli = Cnvrg::CLI.new()
|
281
|
+
commit_sha1 = options[:commit_sha1]
|
282
|
+
cli.list_dataset_commits(dataset_url, commit_sha1:commit_sha1)
|
283
|
+
end
|
284
|
+
|
285
|
+
desc 'data files DATASET_URL', 'Show list of dataset files'
|
286
|
+
method_option :offset, :type => :numeric, :aliases => ["-o", "--offset"], :default => 0
|
287
|
+
method_option :limit, :type => :numeric, :aliases => ["-l", "--limit"], :default => 1000
|
288
|
+
method_option :expires, :type => :numeric, :aliases => ["-ex", "--expires"], :default => 3600
|
289
|
+
method_option :commit_sha1, :type => :string, :aliases => ["-c", "--commit"], :default => nil
|
290
|
+
def files(dataset_url)
|
291
|
+
cli = Cnvrg::CLI.new()
|
292
|
+
cli.verify_logged_in(false)
|
293
|
+
cli.log_start(__method__, args, options)
|
294
|
+
@dataset = Dataset.new(dataset_url: dataset_url)
|
295
|
+
files = @dataset.list_files(
|
296
|
+
commit_sha1: options[:commit_sha1],
|
297
|
+
limit: options[:limit],
|
298
|
+
expires: options[:expires],
|
299
|
+
offset: options[:offset])
|
300
|
+
cli.log_message(files)
|
301
|
+
end
|
302
|
+
|
303
|
+
desc 'data queries', 'List all dataset queries related to current dataset'
|
304
|
+
def queries()
|
305
|
+
cli = Cnvrg::CLI.new()
|
306
|
+
cli.queries()
|
307
|
+
end
|
308
|
+
|
309
|
+
desc 'data query_files QUERY_NAME', 'Show list of all files in specific query'
|
310
|
+
def query_files(query)
|
311
|
+
cli = Cnvrg::CLI.new()
|
312
|
+
cli.query_files(query)
|
313
|
+
end
|
314
|
+
|
315
|
+
desc 'data download_tags_yaml', 'Download dataset tags yml files in current dataset directory'
|
316
|
+
def download_tags_yaml
|
317
|
+
cli = Cnvrg::CLI.new()
|
318
|
+
cli.download_tags_yaml()
|
319
|
+
end
|
320
|
+
|
321
|
+
|
322
|
+
desc 'data test', 'test client'
|
323
|
+
def test(data_url)
|
324
|
+
cli = Cnvrg::CLI.new
|
325
|
+
cli.verify_logged_in(true)
|
326
|
+
cli.log_start(__method__, args, options)
|
327
|
+
@dataset = Dataset.new(dataset_url: data_url)
|
328
|
+
resp = @dataset.get_storage_client
|
329
|
+
@dataset.init_home(remote: false)
|
330
|
+
@dataset.download_softlink
|
331
|
+
|
332
|
+
end
|
333
|
+
|
334
|
+
end
|
335
|
+
end
|