cnvrg 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/bin/cnvrg +5 -0
- data/cnvrg.gemspec +27 -0
- data/lib/cnvrg.rb +10 -0
- data/lib/cnvrg/api.rb +106 -0
- data/lib/cnvrg/auth.rb +74 -0
- data/lib/cnvrg/cli.rb +574 -0
- data/lib/cnvrg/experiment.rb +26 -0
- data/lib/cnvrg/files.rb +115 -0
- data/lib/cnvrg/helpers.rb +90 -0
- data/lib/cnvrg/old_cli.rb +332 -0
- data/lib/cnvrg/project.rb +237 -0
- data/lib/cnvrg/version.rb +3 -0
- metadata +165 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 74b18bfadd766c4b8acdfe5f1278f2c6e4b3b5b5
|
4
|
+
data.tar.gz: 1a9daaed4ca1086589f464bab99747e0dea46840
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 1d5ee9a14308730003e766b1b92a4e30ebcd68e1dad8cdde8525a0ab95601b923f7adacbd74641a68cee34899988a571e3ccc9005a09a945612102dccb76a6d4
|
7
|
+
data.tar.gz: 2a14a3a480372e8c038b9cb5d0a6aa865e844501145785d26e2126971eb3493f87dfc9eec93049f1dc4ba58577fa7a7b8a187e17d27c9394c4efc06aadf67777
|
data/bin/cnvrg
ADDED
data/cnvrg.gemspec
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'cnvrg/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'cnvrg'
|
8
|
+
spec.version = Cnvrg::VERSION
|
9
|
+
spec.authors = ['Yochay Ettun', 'Leah Kolben']
|
10
|
+
spec.email = ['info@cnvrg.io']
|
11
|
+
spec.summary = %q{A CLI tool for interacting with cnvrg.io.}
|
12
|
+
spec.description = %q{A CLI tool for interacting with cnvrg.io.}
|
13
|
+
spec.homepage = 'https://cnvrg.io'
|
14
|
+
|
15
|
+
#spec.files = `git ls-files`.split($/)
|
16
|
+
spec.files = %w[cnvrg.gemspec] + Dir['*.md', 'bin/*', 'lib/**/*.rb']
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.executables = ['cnvrg']
|
19
|
+
spec.require_paths = ['lib']
|
20
|
+
spec.add_runtime_dependency 'mimemagic', '~> 0.3.1','>=0.3.2'
|
21
|
+
spec.add_runtime_dependency 'faraday', '~> 0.10.0'
|
22
|
+
spec.add_runtime_dependency 'netrc', '~> 0.11.0'
|
23
|
+
spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
|
24
|
+
spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
|
25
|
+
spec.add_runtime_dependency 'thor', '~> 0.19.0','>=0.19.1'
|
26
|
+
end
|
27
|
+
|
data/lib/cnvrg.rb
ADDED
data/lib/cnvrg/api.rb
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
require 'netrc'
|
2
|
+
require 'faraday'
|
3
|
+
require 'json'
|
4
|
+
require 'fileutils'
|
5
|
+
|
6
|
+
module Cnvrg
|
7
|
+
class API
|
8
|
+
USER_AGENT = "CnvrgCLI/#{Cnvrg::VERSION}"
|
9
|
+
# ENDPOINT = 'http://localhost:3000/api'
|
10
|
+
ENDPOINT = 'https://cnvrg.io/api'
|
11
|
+
ENDPOINT_VERSION = 'v1'
|
12
|
+
|
13
|
+
def self.request(resource, method = 'GET', data = {}, parse_request = true)
|
14
|
+
begin
|
15
|
+
n = Netrc.read
|
16
|
+
rescue => e
|
17
|
+
puts e.message
|
18
|
+
end
|
19
|
+
|
20
|
+
# Make sure there is an entry for the Acquia API before generating the
|
21
|
+
# requests.
|
22
|
+
if n['cnvrg.io'].nil?
|
23
|
+
puts 'You\'re not logged in'
|
24
|
+
puts 'Please log in via `cnvrg login`'
|
25
|
+
return
|
26
|
+
end
|
27
|
+
|
28
|
+
@user, @pass = n[Cnvrg::Helpers.netrc_domain]
|
29
|
+
|
30
|
+
conn = Faraday.new
|
31
|
+
conn.headers['Auth-Token'] = @pass
|
32
|
+
conn.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
|
33
|
+
|
34
|
+
case method
|
35
|
+
when 'GET'
|
36
|
+
response = conn.get "#{endpoint_uri}/#{resource}"
|
37
|
+
|
38
|
+
if parse_request == true
|
39
|
+
JSON.parse(response.body)
|
40
|
+
else
|
41
|
+
response
|
42
|
+
end
|
43
|
+
when 'POST'
|
44
|
+
response = conn.post "#{endpoint_uri}/#{resource}", data
|
45
|
+
|
46
|
+
if parse_request == true
|
47
|
+
JSON.parse(response.body)
|
48
|
+
else
|
49
|
+
response
|
50
|
+
end
|
51
|
+
when 'POST_FILE'
|
52
|
+
conn = Faraday.new do |fr|
|
53
|
+
fr.headers['Auth-Token'] = @pass
|
54
|
+
fr.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
|
55
|
+
fr.headers["Content-Type"] = "multipart/form-data"
|
56
|
+
|
57
|
+
fr.request :multipart
|
58
|
+
fr.request :url_encoded
|
59
|
+
fr.adapter :net_http
|
60
|
+
end
|
61
|
+
|
62
|
+
|
63
|
+
# what if windows?
|
64
|
+
# data[:file] = Faraday::UploadIO.new(data[:absolute_path], content_type)
|
65
|
+
file_base = File.basename(data[:relative_path])
|
66
|
+
temp_path = ".cnvrg/#{file_base}"
|
67
|
+
FileUtils.touch(temp_path)
|
68
|
+
data[:file] = Faraday::UploadIO.new("#{temp_path}", "plain/text")
|
69
|
+
response = conn.post "#{endpoint_uri}/#{resource}", data
|
70
|
+
FileUtils.rm(temp_path)
|
71
|
+
|
72
|
+
if parse_request == true
|
73
|
+
JSON.parse(response.body)
|
74
|
+
else
|
75
|
+
response
|
76
|
+
end
|
77
|
+
when 'DELETE'
|
78
|
+
response = conn.delete "#{endpoint_uri}/#{resource}", data
|
79
|
+
|
80
|
+
if parse_request == true
|
81
|
+
JSON.parse(response.body)
|
82
|
+
else
|
83
|
+
response
|
84
|
+
end
|
85
|
+
else
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def self.endpoint_uri
|
90
|
+
"#{Cnvrg::API::ENDPOINT}/#{Cnvrg::API::ENDPOINT_VERSION}"
|
91
|
+
end
|
92
|
+
|
93
|
+
|
94
|
+
def self.display_error(response)
|
95
|
+
"Oops, an error occurred! Reason: #{response['message']}"
|
96
|
+
end
|
97
|
+
|
98
|
+
# Internal: Ensure the response returns a HTTP 200.
|
99
|
+
#
|
100
|
+
# If the response status isn't a HTTP 200, we need to find out why. This
|
101
|
+
# helps identify the issues earlier on and will prevent extra API calls
|
102
|
+
# that won't complete.
|
103
|
+
#
|
104
|
+
# Returns false if the response code isn't a HTTP 200.
|
105
|
+
end
|
106
|
+
end
|
data/lib/cnvrg/auth.rb
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
|
2
|
+
module Cnvrg
|
3
|
+
class Auth
|
4
|
+
#include Thor::Actions
|
5
|
+
|
6
|
+
def is_logged_in?
|
7
|
+
n = Netrc.read
|
8
|
+
return false if n[Cnvrg::Helpers.netrc_domain].nil?
|
9
|
+
|
10
|
+
email, token = n[Cnvrg::Helpers.netrc_domain]
|
11
|
+
not (email.empty? or token.empty?)
|
12
|
+
end
|
13
|
+
|
14
|
+
def get_email
|
15
|
+
n = Netrc.read
|
16
|
+
email, token = n[Cnvrg::Helpers.netrc_domain]
|
17
|
+
|
18
|
+
if self.is_logged_in?
|
19
|
+
return email
|
20
|
+
else
|
21
|
+
return nil
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def get_token
|
26
|
+
n = Netrc.read
|
27
|
+
email, token = n[Cnvrg::Helpers.netrc_domain]
|
28
|
+
|
29
|
+
if self.is_logged_in?
|
30
|
+
return token
|
31
|
+
else
|
32
|
+
return nil
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def ask(message)
|
37
|
+
HighLine.new.ask(message)
|
38
|
+
end
|
39
|
+
|
40
|
+
|
41
|
+
def ask_password(message)
|
42
|
+
HighLine.new.ask(message) do |q|
|
43
|
+
q.echo = false
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def sign_in(email, password)
|
48
|
+
url = API.endpoint_uri
|
49
|
+
url = URI.parse(url+ "/users/sign_in")
|
50
|
+
http = Net::HTTP.new(url.host, url.port)
|
51
|
+
|
52
|
+
if url.scheme == 'https'
|
53
|
+
http.use_ssl = true
|
54
|
+
end
|
55
|
+
req = Net::HTTP::Post.new(url.request_uri)
|
56
|
+
|
57
|
+
req.add_field("EMAIL", email)
|
58
|
+
req.add_field("PASSWORD", password)
|
59
|
+
|
60
|
+
response = http.request(req)
|
61
|
+
|
62
|
+
result = JSON.parse(response.body)
|
63
|
+
if result["status"] == 200
|
64
|
+
return result["token"]
|
65
|
+
else
|
66
|
+
return nil
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
|
71
|
+
|
72
|
+
|
73
|
+
end
|
74
|
+
end
|
data/lib/cnvrg/cli.rb
ADDED
@@ -0,0 +1,574 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require "pty"
|
3
|
+
require "open4"
|
4
|
+
require 'netrc'
|
5
|
+
require 'net/http'
|
6
|
+
require 'uri'
|
7
|
+
require 'open-uri'
|
8
|
+
require 'json'
|
9
|
+
require 'yaml'
|
10
|
+
require 'digest' # sha1
|
11
|
+
require "highline/import"
|
12
|
+
require 'socket'
|
13
|
+
include Open4
|
14
|
+
require 'cnvrg/helpers'
|
15
|
+
require 'cnvrg/api'
|
16
|
+
require 'cnvrg/auth'
|
17
|
+
require 'cnvrg/project'
|
18
|
+
require 'cnvrg/files'
|
19
|
+
require 'cnvrg/experiment'
|
20
|
+
# DEV VERSION
|
21
|
+
#
|
22
|
+
module Cnvrg
|
23
|
+
class CLI < Thor
|
24
|
+
map %w[--version -v] => :__print_version
|
25
|
+
|
26
|
+
desc "--version, -v", "print the version"
|
27
|
+
|
28
|
+
def __print_version
|
29
|
+
puts Cnvrg::VERSION
|
30
|
+
end
|
31
|
+
|
32
|
+
|
33
|
+
desc 'login', 'Authenticate with cnvrg.io and store credentials'
|
34
|
+
|
35
|
+
def login
|
36
|
+
cmd = HighLine.new
|
37
|
+
|
38
|
+
say 'Authenticating with cnvrg', Thor::Shell::Color::YELLOW
|
39
|
+
|
40
|
+
@auth = Cnvrg::Auth.new
|
41
|
+
netrc = Netrc.read
|
42
|
+
@email, token = netrc[Cnvrg::Helpers.netrc_domain]
|
43
|
+
|
44
|
+
if @email and token
|
45
|
+
say 'Seems you\'re already logged in', Thor::Shell::Color::BLUE
|
46
|
+
exit(0)
|
47
|
+
end
|
48
|
+
@email = ask("Enter your Email:")
|
49
|
+
password = cmd.ask("Enter your password (hidden):") { |q| q.echo = "*" }
|
50
|
+
|
51
|
+
if (token = @auth.sign_in(@email, password))
|
52
|
+
netrc[Cnvrg::Helpers.netrc_domain] = @email, token
|
53
|
+
netrc.save
|
54
|
+
|
55
|
+
say "Authenticated successfully as #{@email}", Thor::Shell::Color::GREEN
|
56
|
+
|
57
|
+
else
|
58
|
+
say "Failed to authenticate, wrong email/password", Thor::Shell::Color::RED
|
59
|
+
exit false
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
desc 'logout', 'Logout existing user'
|
64
|
+
|
65
|
+
def logout
|
66
|
+
netrc = Netrc.read
|
67
|
+
netrc.delete(Cnvrg::Helpers.netrc_domain)
|
68
|
+
netrc.save
|
69
|
+
say "Logged out successfully.\n", Thor::Shell::Color::GREEN
|
70
|
+
end
|
71
|
+
|
72
|
+
desc 'me', 'Prints the current logged in user email'
|
73
|
+
|
74
|
+
def me
|
75
|
+
verify_logged_in()
|
76
|
+
auth = Cnvrg::Auth.new
|
77
|
+
if (email = auth.get_email)
|
78
|
+
say "Logged in as: #{email}", Thor::Shell::Color::GREEN
|
79
|
+
else
|
80
|
+
say "You're not logged in.", Thor::Shell::Color::RED
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
## Projects
|
85
|
+
|
86
|
+
desc 'new', 'Create a new cnvrg project'
|
87
|
+
method_option :clean, :type => :boolean, :aliases => ["-c", "--c"], :default => false
|
88
|
+
|
89
|
+
def new(project_name)
|
90
|
+
verify_logged_in()
|
91
|
+
clean = options["clean"]
|
92
|
+
say "Creating #{project_name}", Thor::Shell::Color::BLUE
|
93
|
+
if Dir.exists? project_name or File.exists? project_name
|
94
|
+
say "Conflict with dir/file #{project_name}", Thor::Shell::Color::RED
|
95
|
+
exit(1)
|
96
|
+
end
|
97
|
+
|
98
|
+
if Project.create(project_name, clean)
|
99
|
+
path = Dir.pwd + "/" + project_name
|
100
|
+
@project = Project.new(path)
|
101
|
+
@project.generate_idx
|
102
|
+
else
|
103
|
+
say "Error creating project, please contact support.", Thor::Shell::Color::RED
|
104
|
+
exit(0)
|
105
|
+
end
|
106
|
+
|
107
|
+
say "created\t\tproject's tree", Thor::Shell::Color::GREEN
|
108
|
+
say "created\t\tproject's config", Thor::Shell::Color::GREEN
|
109
|
+
say "Linked directory to\t#{@project.url}", Thor::Shell::Color::GREEN
|
110
|
+
end
|
111
|
+
|
112
|
+
desc 'link', 'Link current directory to a cnvrg project'
|
113
|
+
method_option :sync, :type => :boolean, :aliases => ["-s", "--s"], :default => true
|
114
|
+
|
115
|
+
def link
|
116
|
+
verify_logged_in()
|
117
|
+
sync = options["sync"]
|
118
|
+
project_name =File.basename(Dir.getwd)
|
119
|
+
say "Linking #{project_name}", Thor::Shell::Color::BLUE
|
120
|
+
if File.directory?(Dir.getwd+"/.cnvrg")
|
121
|
+
config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
|
122
|
+
say "Directory is already linked to #{config[:project_slug]}", Thor::Shell::Color::RED
|
123
|
+
exit(0)
|
124
|
+
end
|
125
|
+
if Project.link(project_name)
|
126
|
+
path = Dir.pwd
|
127
|
+
@project = Project.new(path)
|
128
|
+
@project.generate_idx()
|
129
|
+
if sync
|
130
|
+
upload(true)
|
131
|
+
end
|
132
|
+
|
133
|
+
url = @project.url
|
134
|
+
say "#{project_name}'s location is: #{url}\n", Thor::Shell::Color::BLUE
|
135
|
+
else
|
136
|
+
say "Error linking project, please contact support.", Thor::Shell::Color::RED
|
137
|
+
exit(0)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
desc 'clone', 'Clone a project'
|
142
|
+
|
143
|
+
def clone(project_url)
|
144
|
+
verify_logged_in()
|
145
|
+
url_parts = project_url.split("/")
|
146
|
+
project_index = Cnvrg::Helpers.look_for_in_path(project_url, "projects")
|
147
|
+
slug = url_parts[project_index+1]
|
148
|
+
owner = url_parts[project_index-1]
|
149
|
+
response = Cnvrg::API.request("users/#{owner}/projects/#{slug}/get_project", 'GET')
|
150
|
+
Cnvrg::CLI.is_response_success(response)
|
151
|
+
response = JSON.parse response["result"]
|
152
|
+
project_name = response["title"]
|
153
|
+
say "Cloning #{project_name}", Thor::Shell::Color::BLUE
|
154
|
+
if Dir.exists? project_name or File.exists? project_name
|
155
|
+
say "Error: Conflict with dir/file #{project_name}", Thor::Shell::Color::RED
|
156
|
+
exit(1)
|
157
|
+
end
|
158
|
+
|
159
|
+
if Project.clone_dir(slug, owner, project_name)
|
160
|
+
project_home = Dir.pwd+"/"+project_name
|
161
|
+
@project = Project.new(project_home)
|
162
|
+
@files = Cnvrg::Files.new(@project.owner, slug)
|
163
|
+
response = @project.clone
|
164
|
+
Cnvrg::CLI.is_response_success response
|
165
|
+
idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
|
166
|
+
File.open(project_name + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
|
167
|
+
successful_changes = []
|
168
|
+
say "Downloading files", Thor::Shell::Color::BLUE
|
169
|
+
response["result"]["tree"].each do |f|
|
170
|
+
relative_path = f[0].gsub(/^#{@project.local_path}/, "")
|
171
|
+
if f[0].end_with? "/"
|
172
|
+
# dir
|
173
|
+
if @files.download_dir(f[0], relative_path, project_home)
|
174
|
+
successful_changes << relative_path
|
175
|
+
end
|
176
|
+
else
|
177
|
+
# blob
|
178
|
+
if @files.download_file(f[0], relative_path, project_home)
|
179
|
+
successful_changes << relative_path
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
183
|
+
say "Done.\nDownloaded total of #{successful_changes.size} files", Thor::Shell::Color::BLUE
|
184
|
+
else
|
185
|
+
say "Error: Couldn't create directory: #{project_name}", Thor::Shell::Color::RED
|
186
|
+
exit(1)
|
187
|
+
end
|
188
|
+
|
189
|
+
end
|
190
|
+
|
191
|
+
desc 'status', 'Show the working tree status'
|
192
|
+
|
193
|
+
def status
|
194
|
+
verify_logged_in()
|
195
|
+
@project = Project.new(get_project_home)
|
196
|
+
result = @project.compare_idx["result"]
|
197
|
+
commit = result["commit"]
|
198
|
+
result = result["tree"]
|
199
|
+
say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE
|
200
|
+
if result["added"].empty? and result["updated_on_local"].empty? and result["updated_on_server"].empty? and result["deleted"].empty? and result["conflicts"].empty?
|
201
|
+
say "Project is up to date", Thor::Shell::Color::GREEN
|
202
|
+
return true
|
203
|
+
end
|
204
|
+
if result["added"].size > 0
|
205
|
+
say "Added files:\n", Thor::Shell::Color::BLUE
|
206
|
+
result["added"].each do |a|
|
207
|
+
say "\t\tA:\t#{a}", Thor::Shell::Color::GREEN
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
if result["deleted"].size > 0
|
212
|
+
say "Deleted files:\n", Thor::Shell::Color::BLUE
|
213
|
+
result["deleted"].each do |a|
|
214
|
+
say "\t\tD:\t#{a}", Thor::Shell::Color::GREEN
|
215
|
+
end
|
216
|
+
end
|
217
|
+
if result["updated_on_local"].size > 0
|
218
|
+
say "Local changes:\n", Thor::Shell::Color::BLUE
|
219
|
+
result["updated_on_local"].each do |a|
|
220
|
+
say "\t\tM:\t#{a}", Thor::Shell::Color::GREEN
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
if result["updated_on_server"].size > 0
|
225
|
+
say "Remote changes:\n", Thor::Shell::Color::BLUE
|
226
|
+
result["updated_on_server"].each do |a|
|
227
|
+
say "\t\tM:\t#{a}", Thor::Shell::Color::GREEN
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
if result["conflicts"].size > 0
|
232
|
+
say "Conflicted changes:\n", Thor::Shell::Color::BLUE
|
233
|
+
result["conflicts"].each do |a|
|
234
|
+
say "\t\tC:\t#{a}", Thor::Shell::Color::RED
|
235
|
+
end
|
236
|
+
end
|
237
|
+
end
|
238
|
+
|
239
|
+
|
240
|
+
desc 'upload', 'Upload updated files'
|
241
|
+
method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
|
242
|
+
|
243
|
+
def upload(link=false, sync=false)
|
244
|
+
|
245
|
+
verify_logged_in()
|
246
|
+
@project = Project.new(get_project_home)
|
247
|
+
|
248
|
+
@files = Cnvrg::Files.new(@project.owner, @project.slug)
|
249
|
+
ignore = options[:ignore] || []
|
250
|
+
if !@project.update_ignore_list(ignore)
|
251
|
+
say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
|
252
|
+
end
|
253
|
+
result = @project.compare_idx
|
254
|
+
commit = result["result"]["commit"]
|
255
|
+
if !link
|
256
|
+
if commit != @project.last_local_commit and !@project.last_local_commit.nil?
|
257
|
+
say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
|
258
|
+
exit(1)
|
259
|
+
end
|
260
|
+
say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE
|
261
|
+
end
|
262
|
+
result = result["result"]["tree"]
|
263
|
+
if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
|
264
|
+
say "Project is up to date", Thor::Shell::Color::GREEN
|
265
|
+
return true
|
266
|
+
end
|
267
|
+
update_count = 0
|
268
|
+
update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
|
269
|
+
successful_updates = []
|
270
|
+
successful_deletions = []
|
271
|
+
if update_total == 1
|
272
|
+
say "Updating #{update_total} file", Thor::Shell::Color::BLUE
|
273
|
+
else
|
274
|
+
say "Updating #{update_total} files", Thor::Shell::Color::BLUE
|
275
|
+
end
|
276
|
+
|
277
|
+
# Start commit
|
278
|
+
|
279
|
+
commit_sha1 = @files.start_commit["result"]["commit_sha1"]
|
280
|
+
|
281
|
+
# upload / update
|
282
|
+
begin
|
283
|
+
(result["added"] + result["updated_on_local"]).each do |f|
|
284
|
+
puts f
|
285
|
+
relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
|
286
|
+
|
287
|
+
if File.directory?(f)
|
288
|
+
resDir = @files.create_dir(f, relative_path, commit_sha1)
|
289
|
+
if resDir
|
290
|
+
update_count += 1
|
291
|
+
successful_updates<< Helpers.checkmark() + " " + relative_path
|
292
|
+
end
|
293
|
+
else
|
294
|
+
res = @files.upload_file(f, relative_path, commit_sha1)
|
295
|
+
if res
|
296
|
+
update_count += 1
|
297
|
+
successful_updates<< Helpers.checkmark() + " " + relative_path
|
298
|
+
else
|
299
|
+
@files.rollback_commit(commit_sha1)
|
300
|
+
say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
|
301
|
+
exit(0)
|
302
|
+
end
|
303
|
+
end
|
304
|
+
end
|
305
|
+
|
306
|
+
# delete
|
307
|
+
result["deleted"].each do |f|
|
308
|
+
relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
|
309
|
+
if relative_path.end_with?("/")
|
310
|
+
if @files.delete_dir(f, relative_path, commit_sha1)
|
311
|
+
update_count += 1
|
312
|
+
successful_updates<< Helpers.checkmark() + " " + relative_path
|
313
|
+
end
|
314
|
+
else
|
315
|
+
if @files.delete_file(f, relative_path, commit_sha1)
|
316
|
+
update_count += 1
|
317
|
+
successful_updates<< Helpers.checkmark() + " " + relative_path
|
318
|
+
end
|
319
|
+
end
|
320
|
+
end
|
321
|
+
|
322
|
+
rescue Interrupt
|
323
|
+
@files.rollback_commit(commit_sha1)
|
324
|
+
say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
|
325
|
+
exit(0)
|
326
|
+
end
|
327
|
+
if update_count == update_total
|
328
|
+
res = @files.end_commit(commit_sha1)
|
329
|
+
if (Cnvrg::CLI.is_response_success(res, false))
|
330
|
+
# save idx
|
331
|
+
@project.update_idx_with_files_commits!((successful_deletions+successful_updates), res["result"]["commit_time"])
|
332
|
+
|
333
|
+
@project.update_idx_with_commit!(commit_sha1)
|
334
|
+
say "Done", Thor::Shell::Color::BLUE
|
335
|
+
if successful_updates.size >0
|
336
|
+
say "Updated:", Thor::Shell::Color::GREEN
|
337
|
+
say successful_updates.join("\n"), Thor::Shell::Color::GREEN
|
338
|
+
end
|
339
|
+
if successful_deletions.size >0
|
340
|
+
say "Deleted:", Thor::Shell::Color::GREEN
|
341
|
+
say successful_deletions.join("\n"), Thor::Shell::Color::GREEN
|
342
|
+
end
|
343
|
+
say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
|
344
|
+
else
|
345
|
+
@files.rollback_commit(commit_sha1)
|
346
|
+
say "Error. Rolling Back all changes.", Thor::Shell::Color::RED
|
347
|
+
end
|
348
|
+
else
|
349
|
+
@files.rollback_commit(commit_sha1)
|
350
|
+
end
|
351
|
+
|
352
|
+
end
|
353
|
+
|
354
|
+
desc 'download', 'Download updated files'
|
355
|
+
|
356
|
+
def download
|
357
|
+
verify_logged_in()
|
358
|
+
project_home = get_project_home
|
359
|
+
@project = Project.new(project_home)
|
360
|
+
@files = Cnvrg::Files.new(@project.owner, @project.slug)
|
361
|
+
|
362
|
+
res = @project.compare_idx["result"]
|
363
|
+
result = res["tree"]
|
364
|
+
commit = res["commit"]
|
365
|
+
if result["updated_on_server"].empty? and result["conflicts"] and result["deleted"].empty?
|
366
|
+
say "Project is up to date", Thor::Shell::Color::GREEN
|
367
|
+
return true
|
368
|
+
end
|
369
|
+
update_count = 0
|
370
|
+
update_total = result["updated_on_server"].size + result["conflicts"].size
|
371
|
+
|
372
|
+
successful_changes = []
|
373
|
+
if update_total ==1
|
374
|
+
say "Downloading #{update_total} file", Thor::Shell::Color::BLUE
|
375
|
+
else
|
376
|
+
say "Downloading #{update_total} files", Thor::Shell::Color::BLUE
|
377
|
+
|
378
|
+
end
|
379
|
+
|
380
|
+
result["conflicts"].each do |f|
|
381
|
+
relative_path = f.gsub(/^#{@project.local_path}/, "")
|
382
|
+
if @files.download_file(f, relative_path, project_home, conflict=true)
|
383
|
+
successful_changes << relative_path
|
384
|
+
end
|
385
|
+
|
386
|
+
end
|
387
|
+
result["updated_on_server"].each do |f|
|
388
|
+
relative_path = f.gsub(/^#{@project.local_path}/, "")
|
389
|
+
if f.end_with? "/"
|
390
|
+
# dir
|
391
|
+
if @files.download_dir(f, relative_path, project_home)
|
392
|
+
successful_changes << relative_path
|
393
|
+
end
|
394
|
+
else
|
395
|
+
# blob
|
396
|
+
if @files.download_file(f, relative_path, project_home)
|
397
|
+
successful_changes << relative_path
|
398
|
+
end
|
399
|
+
end
|
400
|
+
|
401
|
+
end
|
402
|
+
if update_total == successful_changes.size
|
403
|
+
# update idx with latest commit
|
404
|
+
@project.update_idx_with_commit!(commit)
|
405
|
+
say "Done. Downloaded:", Thor::Shell::Color::GREEN
|
406
|
+
say successful_changes.join("\n"), Thor::Shell::Color::GREEN
|
407
|
+
say "Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN
|
408
|
+
end
|
409
|
+
end
|
410
|
+
|
411
|
+
|
412
|
+
desc 'sync', 'Sync with remote server'
|
413
|
+
|
414
|
+
def sync
|
415
|
+
say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
|
416
|
+
invoke :download
|
417
|
+
invoke :upload
|
418
|
+
end
|
419
|
+
|
420
|
+
# Run
|
421
|
+
#
|
422
|
+
desc 'exec CMD', 'Execute a process'
|
423
|
+
method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
|
424
|
+
method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
|
425
|
+
method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
|
426
|
+
method_option :log, :type => :boolean, :aliases => ["-l", "--l"], :default => false
|
427
|
+
|
428
|
+
def exec(*cmd)
|
429
|
+
verify_logged_in()
|
430
|
+
|
431
|
+
project_home = get_project_home
|
432
|
+
@project = Project.new(project_home)
|
433
|
+
sync_before = options["sync_before"]
|
434
|
+
sync_after = options["sync_after"]
|
435
|
+
print_log = options["log"]
|
436
|
+
title = options["title"]
|
437
|
+
if sync_before
|
438
|
+
# Sync before run
|
439
|
+
say "Syncing project before running", Thor::Shell::Color::BLUE
|
440
|
+
say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
|
441
|
+
download()
|
442
|
+
upload()
|
443
|
+
say "Done Syncing", Thor::Shell::Color::BLUE
|
444
|
+
end
|
445
|
+
|
446
|
+
start_commit = @project.last_local_commit
|
447
|
+
cmd = cmd.join("\s")
|
448
|
+
log = []
|
449
|
+
say "Running: #{cmd}", Thor::Shell::Color::BLUE
|
450
|
+
|
451
|
+
@exp = Experiment.new(@project.owner, @project.slug)
|
452
|
+
|
453
|
+
platform = RUBY_PLATFORM
|
454
|
+
machine_name = Socket.gethostname
|
455
|
+
|
456
|
+
@exp.start(cmd, platform, machine_name, start_commit, title)
|
457
|
+
unless @exp.slug.nil?
|
458
|
+
real = Time.now
|
459
|
+
cpu = Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID)
|
460
|
+
exp_success = true
|
461
|
+
memory_total = []
|
462
|
+
cpu_total = []
|
463
|
+
begin
|
464
|
+
|
465
|
+
PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
|
466
|
+
begin
|
467
|
+
stdout.each do |line|
|
468
|
+
cur_time = Time.now
|
469
|
+
monitor = %x{ps aux|awk '{print $2,$3,$4}'|grep #{pid} }
|
470
|
+
monitor_by = monitor.split(" ")
|
471
|
+
memory = monitor_by[2]
|
472
|
+
cpu = monitor_by[1]
|
473
|
+
memory_total << memory.to_f
|
474
|
+
cpu_total << cpu.to_f
|
475
|
+
cur_log = {time: cur_time,
|
476
|
+
message: line,
|
477
|
+
type: "stdout",
|
478
|
+
rss: memory,
|
479
|
+
cpu: cpu,
|
480
|
+
real: Time.now-real}
|
481
|
+
if print_log
|
482
|
+
puts cur_log
|
483
|
+
end
|
484
|
+
|
485
|
+
log << cur_log
|
486
|
+
|
487
|
+
end
|
488
|
+
if stderr
|
489
|
+
stderr.each do |err|
|
490
|
+
log << {time: Time.now, message: err, type: "stderr"}
|
491
|
+
end
|
492
|
+
end
|
493
|
+
Process.wait(pid)
|
494
|
+
rescue Errno::EIO
|
495
|
+
break
|
496
|
+
end
|
497
|
+
end
|
498
|
+
rescue Errno::ENOENT
|
499
|
+
exp_success = false
|
500
|
+
say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
|
501
|
+
rescue PTY::ChildExited
|
502
|
+
exp_success = false
|
503
|
+
puts "The process exited!"
|
504
|
+
end
|
505
|
+
cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
|
506
|
+
memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
|
507
|
+
exit_status = $?.exitstatus
|
508
|
+
if !exp_success
|
509
|
+
end_commit = @project.last_local_commit
|
510
|
+
res = @exp.end(log, exit_status, end_commit,cpu_average,memory_average)
|
511
|
+
say "Experiment has failed", Thor::Shell::Color::RED
|
512
|
+
exit(0)
|
513
|
+
end
|
514
|
+
if sync_after
|
515
|
+
say "Syncing project after running", Thor::Shell::Color::BLUE
|
516
|
+
# Sync after run
|
517
|
+
download()
|
518
|
+
upload()
|
519
|
+
say "Done Syncing", Thor::Shell::Color::BLUE
|
520
|
+
end
|
521
|
+
end_commit = @project.last_local_commit
|
522
|
+
|
523
|
+
res = @exp.end(log, exit_status, end_commit,cpu_average,memory_average)
|
524
|
+
check = Helpers.checkmark()
|
525
|
+
say "#{check} Done. Experiment's result: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
|
526
|
+
else
|
527
|
+
# didnt run
|
528
|
+
end
|
529
|
+
end
|
530
|
+
|
531
|
+
def self.is_response_success(response, should_exit=true)
|
532
|
+
if response["status"]!= 200
|
533
|
+
error = response['message']
|
534
|
+
say("<%= color('Error: #{error}', RED) %>")
|
535
|
+
if should_exit
|
536
|
+
exit(1)
|
537
|
+
else
|
538
|
+
return false
|
539
|
+
end
|
540
|
+
end
|
541
|
+
return true
|
542
|
+
end
|
543
|
+
|
544
|
+
no_tasks do
|
545
|
+
|
546
|
+
|
547
|
+
def get_project_home
|
548
|
+
absolute_path = Dir.pwd
|
549
|
+
dirs = absolute_path.split("/")
|
550
|
+
dirs.pop while not Dir.exists?("#{dirs.join("/")}/.cnvrg") and dirs.size != 0
|
551
|
+
|
552
|
+
if dirs.size == 0
|
553
|
+
say "Couldn't find cnvrg directory. Please start a new project", Thor::Shell::Color::RED
|
554
|
+
exit(1)
|
555
|
+
end
|
556
|
+
return dirs.join("/")
|
557
|
+
end
|
558
|
+
|
559
|
+
def verify_logged_in
|
560
|
+
auth = Cnvrg::Auth.new
|
561
|
+
unless auth.is_logged_in?
|
562
|
+
say 'You\'re not logged in', Thor::Shell::Color::RED
|
563
|
+
say 'Please log in via `cnvrg login`', Thor::Shell::Color::YELLOW
|
564
|
+
exit(1)
|
565
|
+
end
|
566
|
+
end
|
567
|
+
|
568
|
+
|
569
|
+
end
|
570
|
+
end
|
571
|
+
end
|
572
|
+
|
573
|
+
|
574
|
+
|