cnvrg 1.9.5 → 1.9.9.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/cnvrg.gemspec +1 -1
- data/lib/cnvrg/cli.rb +1 -1
- data/lib/cnvrg/cli/library_cli.rb +1 -1
- data/lib/cnvrg/datafiles.rb +3 -3
- data/lib/cnvrg/dataset.rb +1 -1
- data/lib/cnvrg/files.rb +1 -2
- data/lib/cnvrg/helpers/executer.rb +2 -2
- data/lib/cnvrg/job_cli.rb +6 -7
- data/lib/cnvrg/job_ssh.rb +2 -1
- data/lib/cnvrg/version.rb +1 -1
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0616d28375b48a9b06dc81dd02ea8b874d864667eb58128d97f29aa8c4736153
|
4
|
+
data.tar.gz: 5cb79a6f2ee7c6bca1c6c812ac588e4ac2417c7f1059029037cb1bc522209462
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c2094c716ebe523291de7a87d2094266c2f00cf65fc8067114b12d7739ad764553055ba85125c39a7d2571761fffa31ab7cdb057f77da5d6c20ad1a5601128d5
|
7
|
+
data.tar.gz: c6b9ff106b6b48c96e010da2adf00aa4a17538ee93f901bce357d805f10aedc0c58b47ce4226b3bc9a9683b8272691d5868d81779422253d703beead9b440e7b
|
data/cnvrg.gemspec
CHANGED
@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
|
|
31
31
|
spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
|
32
32
|
spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
|
33
33
|
spec.add_runtime_dependency 'thor', '~> 0.19.0','>=0.19.1'
|
34
|
-
spec.add_runtime_dependency 'aws-sdk', '~>
|
34
|
+
spec.add_runtime_dependency 'aws-sdk-s3', '~> 1'
|
35
35
|
spec.add_runtime_dependency 'signet', '~> 0.11.0'
|
36
36
|
spec.add_runtime_dependency 'google-cloud-env', '~> 1.2.1'
|
37
37
|
spec.add_runtime_dependency 'google-cloud-core', '~> 1.3.2'
|
data/lib/cnvrg/cli.rb
CHANGED
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
require 'mimemagic'
|
2
|
-
require 'aws-sdk'
|
2
|
+
require 'aws-sdk-s3'
|
3
3
|
require 'URLcrypt'
|
4
4
|
require 'parallel'
|
5
5
|
require 'fileutils'
|
@@ -330,7 +330,7 @@ module Cnvrg
|
|
330
330
|
|
331
331
|
end
|
332
332
|
|
333
|
-
def upload_multiple_files_optimized(files, commit_sha1, threads:
|
333
|
+
def upload_multiple_files_optimized(files, commit_sha1, threads: 15, chunk_size: 1000, force: false, new_branch: false, prefix: '', partial_commit: nil)
|
334
334
|
cli = CLI.new
|
335
335
|
cli.log_message("Using #{threads} threads with chunk size of #{chunk_size}.", Thor::Shell::Color::GREEN)
|
336
336
|
|
@@ -406,7 +406,7 @@ module Cnvrg
|
|
406
406
|
# Fetch the required files from the server:
|
407
407
|
Parallel.map((file_chunks), in_threads: 10) do |files_chunk|
|
408
408
|
|
409
|
-
tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix)
|
409
|
+
tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads)
|
410
410
|
results = request_upload_files(commit_sha1, tree, force, new_branch, partial_commit)
|
411
411
|
|
412
412
|
# puts "Got #{results['files'].size} files to upload from #{files_chunk.size} files"
|
data/lib/cnvrg/dataset.rb
CHANGED
@@ -541,7 +541,7 @@ module Cnvrg
|
|
541
541
|
"#{url}/#{self.owner}/datasets/#{self.slug}"
|
542
542
|
end
|
543
543
|
|
544
|
-
def generate_chunked_idx(list_files = [], threads:
|
544
|
+
def generate_chunked_idx(list_files = [], threads: 15, prefix: '')
|
545
545
|
tree = {}
|
546
546
|
Parallel.map(list_files, in_threads: threads) do |file|
|
547
547
|
#check if prefix exists do prefix/path otherwise path
|
data/lib/cnvrg/files.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
require 'mimemagic'
|
2
|
-
require 'aws-sdk'
|
2
|
+
require 'aws-sdk-s3'
|
3
3
|
require 'URLcrypt'
|
4
4
|
require 'tempfile'
|
5
5
|
require 'net/http'
|
@@ -727,7 +727,6 @@ module Cnvrg
|
|
727
727
|
unless Cnvrg::CLI.is_response_success(res, false)
|
728
728
|
raise SignalException.new("Cant download files from the server.")
|
729
729
|
end
|
730
|
-
puts res
|
731
730
|
self.download_multpile_files_s3(res['result'], @project_home, postfix: postfix, progress: progress)
|
732
731
|
end
|
733
732
|
|
@@ -89,7 +89,7 @@ class Cnvrg::Helpers::Executer
|
|
89
89
|
end
|
90
90
|
|
91
91
|
def current_homedir
|
92
|
-
`env | grep HOME`.strip.split("=").try(:last)
|
92
|
+
`env | grep -w HOME`.strip.split("=").try(:last)
|
93
93
|
end
|
94
94
|
|
95
95
|
def spark_path
|
@@ -97,7 +97,7 @@ class Cnvrg::Helpers::Executer
|
|
97
97
|
end
|
98
98
|
|
99
99
|
def slave_homedir()
|
100
|
-
run_in_slave("env | grep HOME").split("=").try(:last)
|
100
|
+
run_in_slave("env | grep -w HOME").split("=").try(:last)
|
101
101
|
end
|
102
102
|
|
103
103
|
def slave_env
|
data/lib/cnvrg/job_cli.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
module Cnvrg
|
2
2
|
class JobCli < SubCommandBase
|
3
3
|
|
4
|
-
desc 'log',
|
4
|
+
desc 'log', '', :hide => true
|
5
5
|
method_option :level, :type => :string, :aliases => ["-l", "--level"], :default => 'info'
|
6
6
|
method_option :step, :type => :string, :aliases => ["-s", "--step"], :default => nil
|
7
7
|
method_option :restart, :type => :boolean, :aliases => ["-r", "--restart"], :default => false
|
@@ -15,7 +15,7 @@ module Cnvrg
|
|
15
15
|
end
|
16
16
|
end
|
17
17
|
|
18
|
-
desc 'requirements',
|
18
|
+
desc 'requirements', '', :hide => true
|
19
19
|
def requirements
|
20
20
|
cli = Cnvrg::CLI.new
|
21
21
|
cli.log_start(__method__, args, options)
|
@@ -23,7 +23,7 @@ module Cnvrg
|
|
23
23
|
@project = Project.new(project_dir, owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
|
24
24
|
end
|
25
25
|
|
26
|
-
desc 'install_reqs',
|
26
|
+
desc 'install_reqs', 'Install requirements', :hide => true
|
27
27
|
def install_reqs
|
28
28
|
cli = Cnvrg::CLI.new
|
29
29
|
cli.log_start(__method__, args, options)
|
@@ -33,8 +33,7 @@ module Cnvrg
|
|
33
33
|
@executer.execute_cmds(commands)
|
34
34
|
end
|
35
35
|
|
36
|
-
|
37
|
-
desc 'start', description: "Job Start!", hide: true
|
36
|
+
desc 'start', 'Job Start!', :hide => true
|
38
37
|
def start
|
39
38
|
cli = Cnvrg::CLI.new
|
40
39
|
cli.log_start(__method__, args, options)
|
@@ -47,7 +46,7 @@ module Cnvrg
|
|
47
46
|
@executer.main_thread
|
48
47
|
end
|
49
48
|
|
50
|
-
desc 'stats',
|
49
|
+
desc 'stats', 'stats of agent and slave', :hide => true
|
51
50
|
def stats
|
52
51
|
cli = Cnvrg::CLI.new
|
53
52
|
cli.log_start(__method__, args, options)
|
@@ -59,7 +58,7 @@ module Cnvrg
|
|
59
58
|
@executer.executer_stats
|
60
59
|
end
|
61
60
|
|
62
|
-
desc 'pre_pod_stop',
|
61
|
+
desc 'pre_pod_stop', '', :hide => true
|
63
62
|
def pre_pod_stop
|
64
63
|
cli = Cnvrg::CLI.new
|
65
64
|
cli.log_start(__method__, args, options)
|
data/lib/cnvrg/job_ssh.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
module Cnvrg
|
2
2
|
class JobSsh < SubCommandBase
|
3
|
-
|
3
|
+
|
4
|
+
desc 'ssh start', 'stats of agent and slave'
|
4
5
|
method_option :port, :type => :numeric, :aliases => ["-p", "--port"], :desc => "Port to bind into", :default => 2222
|
5
6
|
method_option :username, :type => :string, :aliases => ["-u", "--username"], :desc => "Job container user name", :default => nil
|
6
7
|
method_option :password, :type => :string, :aliases => ["--password"], :desc =>"Job Conatainer user name, will be set by cnvrg", :default => nil
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.9.
|
4
|
+
version: 1.9.9.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2020-07-
|
13
|
+
date: 2020-07-15 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: bundler
|
@@ -205,19 +205,19 @@ dependencies:
|
|
205
205
|
- !ruby/object:Gem::Version
|
206
206
|
version: 0.19.1
|
207
207
|
- !ruby/object:Gem::Dependency
|
208
|
-
name: aws-sdk
|
208
|
+
name: aws-sdk-s3
|
209
209
|
requirement: !ruby/object:Gem::Requirement
|
210
210
|
requirements:
|
211
211
|
- - "~>"
|
212
212
|
- !ruby/object:Gem::Version
|
213
|
-
version: '
|
213
|
+
version: '1'
|
214
214
|
type: :runtime
|
215
215
|
prerelease: false
|
216
216
|
version_requirements: !ruby/object:Gem::Requirement
|
217
217
|
requirements:
|
218
218
|
- - "~>"
|
219
219
|
- !ruby/object:Gem::Version
|
220
|
-
version: '
|
220
|
+
version: '1'
|
221
221
|
- !ruby/object:Gem::Dependency
|
222
222
|
name: signet
|
223
223
|
requirement: !ruby/object:Gem::Requirement
|
@@ -453,7 +453,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
453
453
|
- !ruby/object:Gem::Version
|
454
454
|
version: '0'
|
455
455
|
requirements: []
|
456
|
-
rubygems_version: 3.
|
456
|
+
rubygems_version: 3.0.4
|
457
457
|
signing_key:
|
458
458
|
specification_version: 4
|
459
459
|
summary: A CLI tool for interacting with cnvrg.io.
|