cnvrg 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/bin/cnvrg +5 -0
- data/cnvrg.gemspec +27 -0
- data/lib/cnvrg.rb +10 -0
- data/lib/cnvrg/api.rb +106 -0
- data/lib/cnvrg/auth.rb +74 -0
- data/lib/cnvrg/cli.rb +574 -0
- data/lib/cnvrg/experiment.rb +26 -0
- data/lib/cnvrg/files.rb +115 -0
- data/lib/cnvrg/helpers.rb +90 -0
- data/lib/cnvrg/old_cli.rb +332 -0
- data/lib/cnvrg/project.rb +237 -0
- data/lib/cnvrg/version.rb +3 -0
- metadata +165 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 74b18bfadd766c4b8acdfe5f1278f2c6e4b3b5b5
|
4
|
+
data.tar.gz: 1a9daaed4ca1086589f464bab99747e0dea46840
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 1d5ee9a14308730003e766b1b92a4e30ebcd68e1dad8cdde8525a0ab95601b923f7adacbd74641a68cee34899988a571e3ccc9005a09a945612102dccb76a6d4
|
7
|
+
data.tar.gz: 2a14a3a480372e8c038b9cb5d0a6aa865e844501145785d26e2126971eb3493f87dfc9eec93049f1dc4ba58577fa7a7b8a187e17d27c9394c4efc06aadf67777
|
data/bin/cnvrg
ADDED
data/cnvrg.gemspec
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'cnvrg/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'cnvrg'
|
8
|
+
spec.version = Cnvrg::VERSION
|
9
|
+
spec.authors = ['Yochay Ettun', 'Leah Kolben']
|
10
|
+
spec.email = ['info@cnvrg.io']
|
11
|
+
spec.summary = %q{A CLI tool for interacting with cnvrg.io.}
|
12
|
+
spec.description = %q{A CLI tool for interacting with cnvrg.io.}
|
13
|
+
spec.homepage = 'https://cnvrg.io'
|
14
|
+
|
15
|
+
#spec.files = `git ls-files`.split($/)
|
16
|
+
spec.files = %w[cnvrg.gemspec] + Dir['*.md', 'bin/*', 'lib/**/*.rb']
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.executables = ['cnvrg']
|
19
|
+
spec.require_paths = ['lib']
|
20
|
+
spec.add_runtime_dependency 'mimemagic', '~> 0.3.1','>=0.3.2'
|
21
|
+
spec.add_runtime_dependency 'faraday', '~> 0.10.0'
|
22
|
+
spec.add_runtime_dependency 'netrc', '~> 0.11.0'
|
23
|
+
spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
|
24
|
+
spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
|
25
|
+
spec.add_runtime_dependency 'thor', '~> 0.19.0','>=0.19.1'
|
26
|
+
end
|
27
|
+
|
data/lib/cnvrg.rb
ADDED
data/lib/cnvrg/api.rb
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
require 'netrc'
|
2
|
+
require 'faraday'
|
3
|
+
require 'json'
|
4
|
+
require 'fileutils'
|
5
|
+
|
6
|
+
module Cnvrg
|
7
|
+
class API
|
8
|
+
USER_AGENT = "CnvrgCLI/#{Cnvrg::VERSION}"
|
9
|
+
# ENDPOINT = 'http://localhost:3000/api'
|
10
|
+
ENDPOINT = 'https://cnvrg.io/api'
|
11
|
+
ENDPOINT_VERSION = 'v1'
|
12
|
+
|
13
|
+
def self.request(resource, method = 'GET', data = {}, parse_request = true)
|
14
|
+
begin
|
15
|
+
n = Netrc.read
|
16
|
+
rescue => e
|
17
|
+
puts e.message
|
18
|
+
end
|
19
|
+
|
20
|
+
# Make sure there is an entry for the Acquia API before generating the
|
21
|
+
# requests.
|
22
|
+
if n['cnvrg.io'].nil?
|
23
|
+
puts 'You\'re not logged in'
|
24
|
+
puts 'Please log in via `cnvrg login`'
|
25
|
+
return
|
26
|
+
end
|
27
|
+
|
28
|
+
@user, @pass = n[Cnvrg::Helpers.netrc_domain]
|
29
|
+
|
30
|
+
conn = Faraday.new
|
31
|
+
conn.headers['Auth-Token'] = @pass
|
32
|
+
conn.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
|
33
|
+
|
34
|
+
case method
|
35
|
+
when 'GET'
|
36
|
+
response = conn.get "#{endpoint_uri}/#{resource}"
|
37
|
+
|
38
|
+
if parse_request == true
|
39
|
+
JSON.parse(response.body)
|
40
|
+
else
|
41
|
+
response
|
42
|
+
end
|
43
|
+
when 'POST'
|
44
|
+
response = conn.post "#{endpoint_uri}/#{resource}", data
|
45
|
+
|
46
|
+
if parse_request == true
|
47
|
+
JSON.parse(response.body)
|
48
|
+
else
|
49
|
+
response
|
50
|
+
end
|
51
|
+
when 'POST_FILE'
|
52
|
+
conn = Faraday.new do |fr|
|
53
|
+
fr.headers['Auth-Token'] = @pass
|
54
|
+
fr.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
|
55
|
+
fr.headers["Content-Type"] = "multipart/form-data"
|
56
|
+
|
57
|
+
fr.request :multipart
|
58
|
+
fr.request :url_encoded
|
59
|
+
fr.adapter :net_http
|
60
|
+
end
|
61
|
+
|
62
|
+
|
63
|
+
# what if windows?
|
64
|
+
# data[:file] = Faraday::UploadIO.new(data[:absolute_path], content_type)
|
65
|
+
file_base = File.basename(data[:relative_path])
|
66
|
+
temp_path = ".cnvrg/#{file_base}"
|
67
|
+
FileUtils.touch(temp_path)
|
68
|
+
data[:file] = Faraday::UploadIO.new("#{temp_path}", "plain/text")
|
69
|
+
response = conn.post "#{endpoint_uri}/#{resource}", data
|
70
|
+
FileUtils.rm(temp_path)
|
71
|
+
|
72
|
+
if parse_request == true
|
73
|
+
JSON.parse(response.body)
|
74
|
+
else
|
75
|
+
response
|
76
|
+
end
|
77
|
+
when 'DELETE'
|
78
|
+
response = conn.delete "#{endpoint_uri}/#{resource}", data
|
79
|
+
|
80
|
+
if parse_request == true
|
81
|
+
JSON.parse(response.body)
|
82
|
+
else
|
83
|
+
response
|
84
|
+
end
|
85
|
+
else
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def self.endpoint_uri
|
90
|
+
"#{Cnvrg::API::ENDPOINT}/#{Cnvrg::API::ENDPOINT_VERSION}"
|
91
|
+
end
|
92
|
+
|
93
|
+
|
94
|
+
def self.display_error(response)
|
95
|
+
"Oops, an error occurred! Reason: #{response['message']}"
|
96
|
+
end
|
97
|
+
|
98
|
+
# Internal: Ensure the response returns a HTTP 200.
|
99
|
+
#
|
100
|
+
# If the response status isn't a HTTP 200, we need to find out why. This
|
101
|
+
# helps identify the issues earlier on and will prevent extra API calls
|
102
|
+
# that won't complete.
|
103
|
+
#
|
104
|
+
# Returns false if the response code isn't a HTTP 200.
|
105
|
+
end
|
106
|
+
end
|
data/lib/cnvrg/auth.rb
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
|
2
|
+
module Cnvrg
|
3
|
+
class Auth
|
4
|
+
#include Thor::Actions
|
5
|
+
|
6
|
+
def is_logged_in?
|
7
|
+
n = Netrc.read
|
8
|
+
return false if n[Cnvrg::Helpers.netrc_domain].nil?
|
9
|
+
|
10
|
+
email, token = n[Cnvrg::Helpers.netrc_domain]
|
11
|
+
not (email.empty? or token.empty?)
|
12
|
+
end
|
13
|
+
|
14
|
+
def get_email
|
15
|
+
n = Netrc.read
|
16
|
+
email, token = n[Cnvrg::Helpers.netrc_domain]
|
17
|
+
|
18
|
+
if self.is_logged_in?
|
19
|
+
return email
|
20
|
+
else
|
21
|
+
return nil
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def get_token
|
26
|
+
n = Netrc.read
|
27
|
+
email, token = n[Cnvrg::Helpers.netrc_domain]
|
28
|
+
|
29
|
+
if self.is_logged_in?
|
30
|
+
return token
|
31
|
+
else
|
32
|
+
return nil
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def ask(message)
|
37
|
+
HighLine.new.ask(message)
|
38
|
+
end
|
39
|
+
|
40
|
+
|
41
|
+
def ask_password(message)
|
42
|
+
HighLine.new.ask(message) do |q|
|
43
|
+
q.echo = false
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def sign_in(email, password)
|
48
|
+
url = API.endpoint_uri
|
49
|
+
url = URI.parse(url+ "/users/sign_in")
|
50
|
+
http = Net::HTTP.new(url.host, url.port)
|
51
|
+
|
52
|
+
if url.scheme == 'https'
|
53
|
+
http.use_ssl = true
|
54
|
+
end
|
55
|
+
req = Net::HTTP::Post.new(url.request_uri)
|
56
|
+
|
57
|
+
req.add_field("EMAIL", email)
|
58
|
+
req.add_field("PASSWORD", password)
|
59
|
+
|
60
|
+
response = http.request(req)
|
61
|
+
|
62
|
+
result = JSON.parse(response.body)
|
63
|
+
if result["status"] == 200
|
64
|
+
return result["token"]
|
65
|
+
else
|
66
|
+
return nil
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
|
71
|
+
|
72
|
+
|
73
|
+
end
|
74
|
+
end
|
data/lib/cnvrg/cli.rb
ADDED
@@ -0,0 +1,574 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require "pty"
|
3
|
+
require "open4"
|
4
|
+
require 'netrc'
|
5
|
+
require 'net/http'
|
6
|
+
require 'uri'
|
7
|
+
require 'open-uri'
|
8
|
+
require 'json'
|
9
|
+
require 'yaml'
|
10
|
+
require 'digest' # sha1
|
11
|
+
require "highline/import"
|
12
|
+
require 'socket'
|
13
|
+
include Open4
|
14
|
+
require 'cnvrg/helpers'
|
15
|
+
require 'cnvrg/api'
|
16
|
+
require 'cnvrg/auth'
|
17
|
+
require 'cnvrg/project'
|
18
|
+
require 'cnvrg/files'
|
19
|
+
require 'cnvrg/experiment'
|
20
|
+
# DEV VERSION
|
21
|
+
#
|
22
|
+
module Cnvrg
|
23
|
+
class CLI < Thor
|
24
|
+
map %w[--version -v] => :__print_version
|
25
|
+
|
26
|
+
desc "--version, -v", "print the version"
|
27
|
+
|
28
|
+
def __print_version
|
29
|
+
puts Cnvrg::VERSION
|
30
|
+
end
|
31
|
+
|
32
|
+
|
33
|
+
desc 'login', 'Authenticate with cnvrg.io and store credentials'
|
34
|
+
|
35
|
+
def login
|
36
|
+
cmd = HighLine.new
|
37
|
+
|
38
|
+
say 'Authenticating with cnvrg', Thor::Shell::Color::YELLOW
|
39
|
+
|
40
|
+
@auth = Cnvrg::Auth.new
|
41
|
+
netrc = Netrc.read
|
42
|
+
@email, token = netrc[Cnvrg::Helpers.netrc_domain]
|
43
|
+
|
44
|
+
if @email and token
|
45
|
+
say 'Seems you\'re already logged in', Thor::Shell::Color::BLUE
|
46
|
+
exit(0)
|
47
|
+
end
|
48
|
+
@email = ask("Enter your Email:")
|
49
|
+
password = cmd.ask("Enter your password (hidden):") { |q| q.echo = "*" }
|
50
|
+
|
51
|
+
if (token = @auth.sign_in(@email, password))
|
52
|
+
netrc[Cnvrg::Helpers.netrc_domain] = @email, token
|
53
|
+
netrc.save
|
54
|
+
|
55
|
+
say "Authenticated successfully as #{@email}", Thor::Shell::Color::GREEN
|
56
|
+
|
57
|
+
else
|
58
|
+
say "Failed to authenticate, wrong email/password", Thor::Shell::Color::RED
|
59
|
+
exit false
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
desc 'logout', 'Logout existing user'
|
64
|
+
|
65
|
+
def logout
|
66
|
+
netrc = Netrc.read
|
67
|
+
netrc.delete(Cnvrg::Helpers.netrc_domain)
|
68
|
+
netrc.save
|
69
|
+
say "Logged out successfully.\n", Thor::Shell::Color::GREEN
|
70
|
+
end
|
71
|
+
|
72
|
+
desc 'me', 'Prints the current logged in user email'
|
73
|
+
|
74
|
+
def me
|
75
|
+
verify_logged_in()
|
76
|
+
auth = Cnvrg::Auth.new
|
77
|
+
if (email = auth.get_email)
|
78
|
+
say "Logged in as: #{email}", Thor::Shell::Color::GREEN
|
79
|
+
else
|
80
|
+
say "You're not logged in.", Thor::Shell::Color::RED
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
## Projects
|
85
|
+
|
86
|
+
desc 'new', 'Create a new cnvrg project'
|
87
|
+
method_option :clean, :type => :boolean, :aliases => ["-c", "--c"], :default => false
|
88
|
+
|
89
|
+
def new(project_name)
|
90
|
+
verify_logged_in()
|
91
|
+
clean = options["clean"]
|
92
|
+
say "Creating #{project_name}", Thor::Shell::Color::BLUE
|
93
|
+
if Dir.exists? project_name or File.exists? project_name
|
94
|
+
say "Conflict with dir/file #{project_name}", Thor::Shell::Color::RED
|
95
|
+
exit(1)
|
96
|
+
end
|
97
|
+
|
98
|
+
if Project.create(project_name, clean)
|
99
|
+
path = Dir.pwd + "/" + project_name
|
100
|
+
@project = Project.new(path)
|
101
|
+
@project.generate_idx
|
102
|
+
else
|
103
|
+
say "Error creating project, please contact support.", Thor::Shell::Color::RED
|
104
|
+
exit(0)
|
105
|
+
end
|
106
|
+
|
107
|
+
say "created\t\tproject's tree", Thor::Shell::Color::GREEN
|
108
|
+
say "created\t\tproject's config", Thor::Shell::Color::GREEN
|
109
|
+
say "Linked directory to\t#{@project.url}", Thor::Shell::Color::GREEN
|
110
|
+
end
|
111
|
+
|
112
|
+
desc 'link', 'Link current directory to a cnvrg project'
|
113
|
+
method_option :sync, :type => :boolean, :aliases => ["-s", "--s"], :default => true
|
114
|
+
|
115
|
+
def link
|
116
|
+
verify_logged_in()
|
117
|
+
sync = options["sync"]
|
118
|
+
project_name =File.basename(Dir.getwd)
|
119
|
+
say "Linking #{project_name}", Thor::Shell::Color::BLUE
|
120
|
+
if File.directory?(Dir.getwd+"/.cnvrg")
|
121
|
+
config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
|
122
|
+
say "Directory is already linked to #{config[:project_slug]}", Thor::Shell::Color::RED
|
123
|
+
exit(0)
|
124
|
+
end
|
125
|
+
if Project.link(project_name)
|
126
|
+
path = Dir.pwd
|
127
|
+
@project = Project.new(path)
|
128
|
+
@project.generate_idx()
|
129
|
+
if sync
|
130
|
+
upload(true)
|
131
|
+
end
|
132
|
+
|
133
|
+
url = @project.url
|
134
|
+
say "#{project_name}'s location is: #{url}\n", Thor::Shell::Color::BLUE
|
135
|
+
else
|
136
|
+
say "Error linking project, please contact support.", Thor::Shell::Color::RED
|
137
|
+
exit(0)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
desc 'clone', 'Clone a project'
|
142
|
+
|
143
|
+
def clone(project_url)
|
144
|
+
verify_logged_in()
|
145
|
+
url_parts = project_url.split("/")
|
146
|
+
project_index = Cnvrg::Helpers.look_for_in_path(project_url, "projects")
|
147
|
+
slug = url_parts[project_index+1]
|
148
|
+
owner = url_parts[project_index-1]
|
149
|
+
response = Cnvrg::API.request("users/#{owner}/projects/#{slug}/get_project", 'GET')
|
150
|
+
Cnvrg::CLI.is_response_success(response)
|
151
|
+
response = JSON.parse response["result"]
|
152
|
+
project_name = response["title"]
|
153
|
+
say "Cloning #{project_name}", Thor::Shell::Color::BLUE
|
154
|
+
if Dir.exists? project_name or File.exists? project_name
|
155
|
+
say "Error: Conflict with dir/file #{project_name}", Thor::Shell::Color::RED
|
156
|
+
exit(1)
|
157
|
+
end
|
158
|
+
|
159
|
+
if Project.clone_dir(slug, owner, project_name)
|
160
|
+
project_home = Dir.pwd+"/"+project_name
|
161
|
+
@project = Project.new(project_home)
|
162
|
+
@files = Cnvrg::Files.new(@project.owner, slug)
|
163
|
+
response = @project.clone
|
164
|
+
Cnvrg::CLI.is_response_success response
|
165
|
+
idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
|
166
|
+
File.open(project_name + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
|
167
|
+
successful_changes = []
|
168
|
+
say "Downloading files", Thor::Shell::Color::BLUE
|
169
|
+
response["result"]["tree"].each do |f|
|
170
|
+
relative_path = f[0].gsub(/^#{@project.local_path}/, "")
|
171
|
+
if f[0].end_with? "/"
|
172
|
+
# dir
|
173
|
+
if @files.download_dir(f[0], relative_path, project_home)
|
174
|
+
successful_changes << relative_path
|
175
|
+
end
|
176
|
+
else
|
177
|
+
# blob
|
178
|
+
if @files.download_file(f[0], relative_path, project_home)
|
179
|
+
successful_changes << relative_path
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
183
|
+
say "Done.\nDownloaded total of #{successful_changes.size} files", Thor::Shell::Color::BLUE
|
184
|
+
else
|
185
|
+
say "Error: Couldn't create directory: #{project_name}", Thor::Shell::Color::RED
|
186
|
+
exit(1)
|
187
|
+
end
|
188
|
+
|
189
|
+
end
|
190
|
+
|
191
|
+
desc 'status', 'Show the working tree status'
|
192
|
+
|
193
|
+
def status
|
194
|
+
verify_logged_in()
|
195
|
+
@project = Project.new(get_project_home)
|
196
|
+
result = @project.compare_idx["result"]
|
197
|
+
commit = result["commit"]
|
198
|
+
result = result["tree"]
|
199
|
+
say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE
|
200
|
+
if result["added"].empty? and result["updated_on_local"].empty? and result["updated_on_server"].empty? and result["deleted"].empty? and result["conflicts"].empty?
|
201
|
+
say "Project is up to date", Thor::Shell::Color::GREEN
|
202
|
+
return true
|
203
|
+
end
|
204
|
+
if result["added"].size > 0
|
205
|
+
say "Added files:\n", Thor::Shell::Color::BLUE
|
206
|
+
result["added"].each do |a|
|
207
|
+
say "\t\tA:\t#{a}", Thor::Shell::Color::GREEN
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
if result["deleted"].size > 0
|
212
|
+
say "Deleted files:\n", Thor::Shell::Color::BLUE
|
213
|
+
result["deleted"].each do |a|
|
214
|
+
say "\t\tD:\t#{a}", Thor::Shell::Color::GREEN
|
215
|
+
end
|
216
|
+
end
|
217
|
+
if result["updated_on_local"].size > 0
|
218
|
+
say "Local changes:\n", Thor::Shell::Color::BLUE
|
219
|
+
result["updated_on_local"].each do |a|
|
220
|
+
say "\t\tM:\t#{a}", Thor::Shell::Color::GREEN
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
if result["updated_on_server"].size > 0
|
225
|
+
say "Remote changes:\n", Thor::Shell::Color::BLUE
|
226
|
+
result["updated_on_server"].each do |a|
|
227
|
+
say "\t\tM:\t#{a}", Thor::Shell::Color::GREEN
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
if result["conflicts"].size > 0
|
232
|
+
say "Conflicted changes:\n", Thor::Shell::Color::BLUE
|
233
|
+
result["conflicts"].each do |a|
|
234
|
+
say "\t\tC:\t#{a}", Thor::Shell::Color::RED
|
235
|
+
end
|
236
|
+
end
|
237
|
+
end
|
238
|
+
|
239
|
+
|
240
|
+
desc 'upload', 'Upload updated files'
|
241
|
+
method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
|
242
|
+
|
243
|
+
def upload(link=false, sync=false)
|
244
|
+
|
245
|
+
verify_logged_in()
|
246
|
+
@project = Project.new(get_project_home)
|
247
|
+
|
248
|
+
@files = Cnvrg::Files.new(@project.owner, @project.slug)
|
249
|
+
ignore = options[:ignore] || []
|
250
|
+
if !@project.update_ignore_list(ignore)
|
251
|
+
say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
|
252
|
+
end
|
253
|
+
result = @project.compare_idx
|
254
|
+
commit = result["result"]["commit"]
|
255
|
+
if !link
|
256
|
+
if commit != @project.last_local_commit and !@project.last_local_commit.nil?
|
257
|
+
say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
|
258
|
+
exit(1)
|
259
|
+
end
|
260
|
+
say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE
|
261
|
+
end
|
262
|
+
result = result["result"]["tree"]
|
263
|
+
if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
|
264
|
+
say "Project is up to date", Thor::Shell::Color::GREEN
|
265
|
+
return true
|
266
|
+
end
|
267
|
+
update_count = 0
|
268
|
+
update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
|
269
|
+
successful_updates = []
|
270
|
+
successful_deletions = []
|
271
|
+
if update_total == 1
|
272
|
+
say "Updating #{update_total} file", Thor::Shell::Color::BLUE
|
273
|
+
else
|
274
|
+
say "Updating #{update_total} files", Thor::Shell::Color::BLUE
|
275
|
+
end
|
276
|
+
|
277
|
+
# Start commit
|
278
|
+
|
279
|
+
commit_sha1 = @files.start_commit["result"]["commit_sha1"]
|
280
|
+
|
281
|
+
# upload / update
|
282
|
+
begin
|
283
|
+
(result["added"] + result["updated_on_local"]).each do |f|
|
284
|
+
puts f
|
285
|
+
relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
|
286
|
+
|
287
|
+
if File.directory?(f)
|
288
|
+
resDir = @files.create_dir(f, relative_path, commit_sha1)
|
289
|
+
if resDir
|
290
|
+
update_count += 1
|
291
|
+
successful_updates<< Helpers.checkmark() + " " + relative_path
|
292
|
+
end
|
293
|
+
else
|
294
|
+
res = @files.upload_file(f, relative_path, commit_sha1)
|
295
|
+
if res
|
296
|
+
update_count += 1
|
297
|
+
successful_updates<< Helpers.checkmark() + " " + relative_path
|
298
|
+
else
|
299
|
+
@files.rollback_commit(commit_sha1)
|
300
|
+
say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
|
301
|
+
exit(0)
|
302
|
+
end
|
303
|
+
end
|
304
|
+
end
|
305
|
+
|
306
|
+
# delete
|
307
|
+
result["deleted"].each do |f|
|
308
|
+
relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
|
309
|
+
if relative_path.end_with?("/")
|
310
|
+
if @files.delete_dir(f, relative_path, commit_sha1)
|
311
|
+
update_count += 1
|
312
|
+
successful_updates<< Helpers.checkmark() + " " + relative_path
|
313
|
+
end
|
314
|
+
else
|
315
|
+
if @files.delete_file(f, relative_path, commit_sha1)
|
316
|
+
update_count += 1
|
317
|
+
successful_updates<< Helpers.checkmark() + " " + relative_path
|
318
|
+
end
|
319
|
+
end
|
320
|
+
end
|
321
|
+
|
322
|
+
rescue Interrupt
|
323
|
+
@files.rollback_commit(commit_sha1)
|
324
|
+
say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
|
325
|
+
exit(0)
|
326
|
+
end
|
327
|
+
if update_count == update_total
|
328
|
+
res = @files.end_commit(commit_sha1)
|
329
|
+
if (Cnvrg::CLI.is_response_success(res, false))
|
330
|
+
# save idx
|
331
|
+
@project.update_idx_with_files_commits!((successful_deletions+successful_updates), res["result"]["commit_time"])
|
332
|
+
|
333
|
+
@project.update_idx_with_commit!(commit_sha1)
|
334
|
+
say "Done", Thor::Shell::Color::BLUE
|
335
|
+
if successful_updates.size >0
|
336
|
+
say "Updated:", Thor::Shell::Color::GREEN
|
337
|
+
say successful_updates.join("\n"), Thor::Shell::Color::GREEN
|
338
|
+
end
|
339
|
+
if successful_deletions.size >0
|
340
|
+
say "Deleted:", Thor::Shell::Color::GREEN
|
341
|
+
say successful_deletions.join("\n"), Thor::Shell::Color::GREEN
|
342
|
+
end
|
343
|
+
say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
|
344
|
+
else
|
345
|
+
@files.rollback_commit(commit_sha1)
|
346
|
+
say "Error. Rolling Back all changes.", Thor::Shell::Color::RED
|
347
|
+
end
|
348
|
+
else
|
349
|
+
@files.rollback_commit(commit_sha1)
|
350
|
+
end
|
351
|
+
|
352
|
+
end
|
353
|
+
|
354
|
+
desc 'download', 'Download updated files'
|
355
|
+
|
356
|
+
def download
|
357
|
+
verify_logged_in()
|
358
|
+
project_home = get_project_home
|
359
|
+
@project = Project.new(project_home)
|
360
|
+
@files = Cnvrg::Files.new(@project.owner, @project.slug)
|
361
|
+
|
362
|
+
res = @project.compare_idx["result"]
|
363
|
+
result = res["tree"]
|
364
|
+
commit = res["commit"]
|
365
|
+
if result["updated_on_server"].empty? and result["conflicts"] and result["deleted"].empty?
|
366
|
+
say "Project is up to date", Thor::Shell::Color::GREEN
|
367
|
+
return true
|
368
|
+
end
|
369
|
+
update_count = 0
|
370
|
+
update_total = result["updated_on_server"].size + result["conflicts"].size
|
371
|
+
|
372
|
+
successful_changes = []
|
373
|
+
if update_total ==1
|
374
|
+
say "Downloading #{update_total} file", Thor::Shell::Color::BLUE
|
375
|
+
else
|
376
|
+
say "Downloading #{update_total} files", Thor::Shell::Color::BLUE
|
377
|
+
|
378
|
+
end
|
379
|
+
|
380
|
+
result["conflicts"].each do |f|
|
381
|
+
relative_path = f.gsub(/^#{@project.local_path}/, "")
|
382
|
+
if @files.download_file(f, relative_path, project_home, conflict=true)
|
383
|
+
successful_changes << relative_path
|
384
|
+
end
|
385
|
+
|
386
|
+
end
|
387
|
+
result["updated_on_server"].each do |f|
|
388
|
+
relative_path = f.gsub(/^#{@project.local_path}/, "")
|
389
|
+
if f.end_with? "/"
|
390
|
+
# dir
|
391
|
+
if @files.download_dir(f, relative_path, project_home)
|
392
|
+
successful_changes << relative_path
|
393
|
+
end
|
394
|
+
else
|
395
|
+
# blob
|
396
|
+
if @files.download_file(f, relative_path, project_home)
|
397
|
+
successful_changes << relative_path
|
398
|
+
end
|
399
|
+
end
|
400
|
+
|
401
|
+
end
|
402
|
+
if update_total == successful_changes.size
|
403
|
+
# update idx with latest commit
|
404
|
+
@project.update_idx_with_commit!(commit)
|
405
|
+
say "Done. Downloaded:", Thor::Shell::Color::GREEN
|
406
|
+
say successful_changes.join("\n"), Thor::Shell::Color::GREEN
|
407
|
+
say "Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN
|
408
|
+
end
|
409
|
+
end
|
410
|
+
|
411
|
+
|
412
|
+
desc 'sync', 'Sync with remote server'
|
413
|
+
|
414
|
+
def sync
|
415
|
+
say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
|
416
|
+
invoke :download
|
417
|
+
invoke :upload
|
418
|
+
end
|
419
|
+
|
420
|
+
# Run
|
421
|
+
#
|
422
|
+
desc 'exec CMD', 'Execute a process'
|
423
|
+
method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
|
424
|
+
method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
|
425
|
+
method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
|
426
|
+
method_option :log, :type => :boolean, :aliases => ["-l", "--l"], :default => false
|
427
|
+
|
428
|
+
def exec(*cmd)
|
429
|
+
verify_logged_in()
|
430
|
+
|
431
|
+
project_home = get_project_home
|
432
|
+
@project = Project.new(project_home)
|
433
|
+
sync_before = options["sync_before"]
|
434
|
+
sync_after = options["sync_after"]
|
435
|
+
print_log = options["log"]
|
436
|
+
title = options["title"]
|
437
|
+
if sync_before
|
438
|
+
# Sync before run
|
439
|
+
say "Syncing project before running", Thor::Shell::Color::BLUE
|
440
|
+
say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
|
441
|
+
download()
|
442
|
+
upload()
|
443
|
+
say "Done Syncing", Thor::Shell::Color::BLUE
|
444
|
+
end
|
445
|
+
|
446
|
+
start_commit = @project.last_local_commit
|
447
|
+
cmd = cmd.join("\s")
|
448
|
+
log = []
|
449
|
+
say "Running: #{cmd}", Thor::Shell::Color::BLUE
|
450
|
+
|
451
|
+
@exp = Experiment.new(@project.owner, @project.slug)
|
452
|
+
|
453
|
+
platform = RUBY_PLATFORM
|
454
|
+
machine_name = Socket.gethostname
|
455
|
+
|
456
|
+
@exp.start(cmd, platform, machine_name, start_commit, title)
|
457
|
+
unless @exp.slug.nil?
|
458
|
+
real = Time.now
|
459
|
+
cpu = Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID)
|
460
|
+
exp_success = true
|
461
|
+
memory_total = []
|
462
|
+
cpu_total = []
|
463
|
+
begin
|
464
|
+
|
465
|
+
PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
|
466
|
+
begin
|
467
|
+
stdout.each do |line|
|
468
|
+
cur_time = Time.now
|
469
|
+
monitor = %x{ps aux|awk '{print $2,$3,$4}'|grep #{pid} }
|
470
|
+
monitor_by = monitor.split(" ")
|
471
|
+
memory = monitor_by[2]
|
472
|
+
cpu = monitor_by[1]
|
473
|
+
memory_total << memory.to_f
|
474
|
+
cpu_total << cpu.to_f
|
475
|
+
cur_log = {time: cur_time,
|
476
|
+
message: line,
|
477
|
+
type: "stdout",
|
478
|
+
rss: memory,
|
479
|
+
cpu: cpu,
|
480
|
+
real: Time.now-real}
|
481
|
+
if print_log
|
482
|
+
puts cur_log
|
483
|
+
end
|
484
|
+
|
485
|
+
log << cur_log
|
486
|
+
|
487
|
+
end
|
488
|
+
if stderr
|
489
|
+
stderr.each do |err|
|
490
|
+
log << {time: Time.now, message: err, type: "stderr"}
|
491
|
+
end
|
492
|
+
end
|
493
|
+
Process.wait(pid)
|
494
|
+
rescue Errno::EIO
|
495
|
+
break
|
496
|
+
end
|
497
|
+
end
|
498
|
+
rescue Errno::ENOENT
|
499
|
+
exp_success = false
|
500
|
+
say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
|
501
|
+
rescue PTY::ChildExited
|
502
|
+
exp_success = false
|
503
|
+
puts "The process exited!"
|
504
|
+
end
|
505
|
+
cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
|
506
|
+
memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
|
507
|
+
exit_status = $?.exitstatus
|
508
|
+
if !exp_success
|
509
|
+
end_commit = @project.last_local_commit
|
510
|
+
res = @exp.end(log, exit_status, end_commit,cpu_average,memory_average)
|
511
|
+
say "Experiment has failed", Thor::Shell::Color::RED
|
512
|
+
exit(0)
|
513
|
+
end
|
514
|
+
if sync_after
|
515
|
+
say "Syncing project after running", Thor::Shell::Color::BLUE
|
516
|
+
# Sync after run
|
517
|
+
download()
|
518
|
+
upload()
|
519
|
+
say "Done Syncing", Thor::Shell::Color::BLUE
|
520
|
+
end
|
521
|
+
end_commit = @project.last_local_commit
|
522
|
+
|
523
|
+
res = @exp.end(log, exit_status, end_commit,cpu_average,memory_average)
|
524
|
+
check = Helpers.checkmark()
|
525
|
+
say "#{check} Done. Experiment's result: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
|
526
|
+
else
|
527
|
+
# didnt run
|
528
|
+
end
|
529
|
+
end
|
530
|
+
|
531
|
+
def self.is_response_success(response, should_exit=true)
|
532
|
+
if response["status"]!= 200
|
533
|
+
error = response['message']
|
534
|
+
say("<%= color('Error: #{error}', RED) %>")
|
535
|
+
if should_exit
|
536
|
+
exit(1)
|
537
|
+
else
|
538
|
+
return false
|
539
|
+
end
|
540
|
+
end
|
541
|
+
return true
|
542
|
+
end
|
543
|
+
|
544
|
+
no_tasks do
|
545
|
+
|
546
|
+
|
547
|
+
def get_project_home
|
548
|
+
absolute_path = Dir.pwd
|
549
|
+
dirs = absolute_path.split("/")
|
550
|
+
dirs.pop while not Dir.exists?("#{dirs.join("/")}/.cnvrg") and dirs.size != 0
|
551
|
+
|
552
|
+
if dirs.size == 0
|
553
|
+
say "Couldn't find cnvrg directory. Please start a new project", Thor::Shell::Color::RED
|
554
|
+
exit(1)
|
555
|
+
end
|
556
|
+
return dirs.join("/")
|
557
|
+
end
|
558
|
+
|
559
|
+
def verify_logged_in
|
560
|
+
auth = Cnvrg::Auth.new
|
561
|
+
unless auth.is_logged_in?
|
562
|
+
say 'You\'re not logged in', Thor::Shell::Color::RED
|
563
|
+
say 'Please log in via `cnvrg login`', Thor::Shell::Color::YELLOW
|
564
|
+
exit(1)
|
565
|
+
end
|
566
|
+
end
|
567
|
+
|
568
|
+
|
569
|
+
end
|
570
|
+
end
|
571
|
+
end
|
572
|
+
|
573
|
+
|
574
|
+
|