cnvrg 1.9.6 → 1.9.9.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +1 -1
- data/lib/cnvrg/cli/library_cli.rb +1 -1
- data/lib/cnvrg/datafiles.rb +7 -7
- data/lib/cnvrg/dataset.rb +1 -1
- data/lib/cnvrg/files.rb +1 -2
- data/lib/cnvrg/helpers/executer.rb +2 -2
- data/lib/cnvrg/job_cli.rb +6 -7
- data/lib/cnvrg/job_ssh.rb +2 -1
- data/lib/cnvrg/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 53ddc2741cb68ba2e0a7952883931cbfc6fc6d0c92ba2a98cd99c3d4bac350ef
|
4
|
+
data.tar.gz: 17a78f413fa1a65027826cf9d4a411176c938e9c260eae5d7813c19c9b6e5e56
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c7bc53a1b26f86dc619a80c6bde93e93908916f665dd2e41742c1645cb855ac3b7f36ee1a32ed324204435ca18a6a6f6b4330cede901d1ae6a149affbb14f6a4
|
7
|
+
data.tar.gz: b17527598197acdd1bc0f00b0f461dc38bb621674ffd2ac144cedc68a4b64097fc898c4b9f4f6c562657e656d1cfdcaa79acdae53ec00d7016d85bee2252f519
|
data/lib/cnvrg/cli.rb
CHANGED
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
require 'mimemagic'
|
2
|
-
require 'aws-sdk'
|
2
|
+
require 'aws-sdk-s3'
|
3
3
|
require 'URLcrypt'
|
4
4
|
require 'parallel'
|
5
5
|
require 'fileutils'
|
@@ -330,7 +330,7 @@ module Cnvrg
|
|
330
330
|
|
331
331
|
end
|
332
332
|
|
333
|
-
def upload_multiple_files_optimized(files, commit_sha1, threads:
|
333
|
+
def upload_multiple_files_optimized(files, commit_sha1, threads: 15, chunk_size: 1000, force: false, new_branch: false, prefix: '', partial_commit: nil)
|
334
334
|
cli = CLI.new
|
335
335
|
cli.log_message("Using #{threads} threads with chunk size of #{chunk_size}.", Thor::Shell::Color::GREEN)
|
336
336
|
|
@@ -406,7 +406,7 @@ module Cnvrg
|
|
406
406
|
# Fetch the required files from the server:
|
407
407
|
Parallel.map((file_chunks), in_threads: 10) do |files_chunk|
|
408
408
|
|
409
|
-
tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix)
|
409
|
+
tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads)
|
410
410
|
results = request_upload_files(commit_sha1, tree, force, new_branch, partial_commit)
|
411
411
|
|
412
412
|
# puts "Got #{results['files'].size} files to upload from #{files_chunk.size} files"
|
@@ -1319,10 +1319,10 @@ module Cnvrg
|
|
1319
1319
|
# blob
|
1320
1320
|
local_path = "#{local_path}.conflict" if conflict
|
1321
1321
|
storage_path = f["path"]
|
1322
|
-
if File.exists? local_path
|
1323
|
-
|
1324
|
-
|
1325
|
-
end
|
1322
|
+
# if File.exists? local_path
|
1323
|
+
# Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
|
1324
|
+
# next
|
1325
|
+
# end
|
1326
1326
|
resp = @downloader.download(storage_path, local_path)
|
1327
1327
|
Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
|
1328
1328
|
rescue => e
|
data/lib/cnvrg/dataset.rb
CHANGED
@@ -541,7 +541,7 @@ module Cnvrg
|
|
541
541
|
"#{url}/#{self.owner}/datasets/#{self.slug}"
|
542
542
|
end
|
543
543
|
|
544
|
-
def generate_chunked_idx(list_files = [], threads:
|
544
|
+
def generate_chunked_idx(list_files = [], threads: 15, prefix: '')
|
545
545
|
tree = {}
|
546
546
|
Parallel.map(list_files, in_threads: threads) do |file|
|
547
547
|
#check if prefix exists do prefix/path otherwise path
|
data/lib/cnvrg/files.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
require 'mimemagic'
|
2
|
-
require 'aws-sdk'
|
2
|
+
require 'aws-sdk-s3'
|
3
3
|
require 'URLcrypt'
|
4
4
|
require 'tempfile'
|
5
5
|
require 'net/http'
|
@@ -727,7 +727,6 @@ module Cnvrg
|
|
727
727
|
unless Cnvrg::CLI.is_response_success(res, false)
|
728
728
|
raise SignalException.new("Cant download files from the server.")
|
729
729
|
end
|
730
|
-
puts res
|
731
730
|
self.download_multpile_files_s3(res['result'], @project_home, postfix: postfix, progress: progress)
|
732
731
|
end
|
733
732
|
|
@@ -89,7 +89,7 @@ class Cnvrg::Helpers::Executer
|
|
89
89
|
end
|
90
90
|
|
91
91
|
def current_homedir
|
92
|
-
`env | grep HOME`.strip.split("=").try(:last)
|
92
|
+
`env | grep -w HOME`.strip.split("=").try(:last)
|
93
93
|
end
|
94
94
|
|
95
95
|
def spark_path
|
@@ -97,7 +97,7 @@ class Cnvrg::Helpers::Executer
|
|
97
97
|
end
|
98
98
|
|
99
99
|
def slave_homedir()
|
100
|
-
run_in_slave("env | grep HOME").split("=").try(:last)
|
100
|
+
run_in_slave("env | grep -w HOME").split("=").try(:last)
|
101
101
|
end
|
102
102
|
|
103
103
|
def slave_env
|
data/lib/cnvrg/job_cli.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
module Cnvrg
|
2
2
|
class JobCli < SubCommandBase
|
3
3
|
|
4
|
-
desc 'log',
|
4
|
+
desc 'log', '', :hide => true
|
5
5
|
method_option :level, :type => :string, :aliases => ["-l", "--level"], :default => 'info'
|
6
6
|
method_option :step, :type => :string, :aliases => ["-s", "--step"], :default => nil
|
7
7
|
method_option :restart, :type => :boolean, :aliases => ["-r", "--restart"], :default => false
|
@@ -15,7 +15,7 @@ module Cnvrg
|
|
15
15
|
end
|
16
16
|
end
|
17
17
|
|
18
|
-
desc 'requirements',
|
18
|
+
desc 'requirements', '', :hide => true
|
19
19
|
def requirements
|
20
20
|
cli = Cnvrg::CLI.new
|
21
21
|
cli.log_start(__method__, args, options)
|
@@ -23,7 +23,7 @@ module Cnvrg
|
|
23
23
|
@project = Project.new(project_dir, owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
|
24
24
|
end
|
25
25
|
|
26
|
-
desc 'install_reqs',
|
26
|
+
desc 'install_reqs', 'Install requirements', :hide => true
|
27
27
|
def install_reqs
|
28
28
|
cli = Cnvrg::CLI.new
|
29
29
|
cli.log_start(__method__, args, options)
|
@@ -33,8 +33,7 @@ module Cnvrg
|
|
33
33
|
@executer.execute_cmds(commands)
|
34
34
|
end
|
35
35
|
|
36
|
-
|
37
|
-
desc 'start', description: "Job Start!", hide: true
|
36
|
+
desc 'start', 'Job Start!', :hide => true
|
38
37
|
def start
|
39
38
|
cli = Cnvrg::CLI.new
|
40
39
|
cli.log_start(__method__, args, options)
|
@@ -47,7 +46,7 @@ module Cnvrg
|
|
47
46
|
@executer.main_thread
|
48
47
|
end
|
49
48
|
|
50
|
-
desc 'stats',
|
49
|
+
desc 'stats', 'stats of agent and slave', :hide => true
|
51
50
|
def stats
|
52
51
|
cli = Cnvrg::CLI.new
|
53
52
|
cli.log_start(__method__, args, options)
|
@@ -59,7 +58,7 @@ module Cnvrg
|
|
59
58
|
@executer.executer_stats
|
60
59
|
end
|
61
60
|
|
62
|
-
desc 'pre_pod_stop',
|
61
|
+
desc 'pre_pod_stop', '', :hide => true
|
63
62
|
def pre_pod_stop
|
64
63
|
cli = Cnvrg::CLI.new
|
65
64
|
cli.log_start(__method__, args, options)
|
data/lib/cnvrg/job_ssh.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
module Cnvrg
|
2
2
|
class JobSsh < SubCommandBase
|
3
|
-
|
3
|
+
|
4
|
+
desc 'ssh start', 'stats of agent and slave'
|
4
5
|
method_option :port, :type => :numeric, :aliases => ["-p", "--port"], :desc => "Port to bind into", :default => 2222
|
5
6
|
method_option :username, :type => :string, :aliases => ["-u", "--username"], :desc => "Job container user name", :default => nil
|
6
7
|
method_option :password, :type => :string, :aliases => ["--password"], :desc =>"Job Conatainer user name, will be set by cnvrg", :default => nil
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.9.
|
4
|
+
version: 1.9.9.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2020-07-
|
13
|
+
date: 2020-07-15 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: bundler
|