rbbt-util 5.26.78 → 5.26.79
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/etc/app.d/base.rb +5 -1
- data/lib/rbbt/resource/path.rb +1 -0
- data/lib/rbbt/resource/util.rb +1 -1
- data/lib/rbbt/resource.rb +1 -0
- data/lib/rbbt/workflow/step.rb +14 -0
- data/lib/rbbt/workflow/util/archive.rb +125 -6
- data/share/rbbt_commands/app/start +4 -1
- data/share/rbbt_commands/migrate_job +1 -96
- data/share/rbbt_commands/purge_job +36 -0
- data/share/rbbt_commands/workflow/server +3 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d7b3f571fcdf2bf96368174818464415b1282a2d08e71fda696d325b1d2a84f4
|
4
|
+
data.tar.gz: f0403774806dc37731e0cfe77bdbdc7908bc6aee9015b657871c786bbd1f849c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '0988f55f748f4917507a819e0bc367bb307893cdc18654bd3af040f4a4f1ade4b64656802c388bfeb0eb9e297e20e1392db869ff4fe9d81871eaccfbf2893e3a'
|
7
|
+
data.tar.gz: e0e865fc6e92866da62eef19bdbdb7fe3dd4036e6fc3f730c01f6823dd5d970477c568d7451bdeab5be76ace260552a19be2d04661a33da79630750fd4fb3143
|
data/etc/app.d/base.rb
CHANGED
@@ -3,13 +3,17 @@ register Sinatra::MultiRoute
|
|
3
3
|
|
4
4
|
register Sinatra::RbbtRESTMain
|
5
5
|
register Sinatra::RbbtRESTEntity
|
6
|
-
register Sinatra::RbbtRESTFileServer # Remove to prevent serving files
|
7
6
|
register Sinatra::RbbtRESTKnowledgeBase
|
8
7
|
register Sinatra::RbbtRESTWorkflow
|
9
8
|
|
10
9
|
helpers Sinatra::RbbtToolHelper
|
11
10
|
helpers Sinatra::RbbtMiscHelpers
|
12
11
|
|
12
|
+
if ENV["RBBT_REST_FILE_SERVER"] == 'true'
|
13
|
+
Log.high "Activating File Server"
|
14
|
+
register Sinatra::RbbtRESTFileServer
|
15
|
+
end
|
16
|
+
|
13
17
|
#{{{ SESSIONS
|
14
18
|
|
15
19
|
use Rack::Session::Cookie, :key => 'rack.session',
|
data/lib/rbbt/resource/path.rb
CHANGED
data/lib/rbbt/resource/util.rb
CHANGED
@@ -25,7 +25,7 @@ module Path
|
|
25
25
|
SLASH = "/"[0]
|
26
26
|
DOT = "."[0]
|
27
27
|
def located?
|
28
|
-
self.byte(0) == SLASH
|
28
|
+
self.byte(0) == SLASH || (self.byte(0) == DOT && self.byte(1) == SLASH) || (resource != Rbbt && (Open.remote?(self) || Open.ssh?(self)))
|
29
29
|
end
|
30
30
|
end
|
31
31
|
|
data/lib/rbbt/resource.rb
CHANGED
data/lib/rbbt/workflow/step.rb
CHANGED
@@ -83,6 +83,20 @@ class Step
|
|
83
83
|
end
|
84
84
|
end
|
85
85
|
|
86
|
+
def load_dependencies_from_info
|
87
|
+
@dependencies = (self.info[:dependencies] || []).collect do |task,name,dep_path|
|
88
|
+
if Open.exists?(dep_path) || Open.exists?(dep_path + '.info')
|
89
|
+
Workflow._load_step dep_path
|
90
|
+
else
|
91
|
+
next if FalseClass === relocated
|
92
|
+
new_path = Workflow.relocate(path, dep_path)
|
93
|
+
relocated = true if Open.exists?(new_path) || Open.exists?(new_path + '.info')
|
94
|
+
Workflow._load_step new_path
|
95
|
+
end
|
96
|
+
end.compact
|
97
|
+
@relocated = relocated
|
98
|
+
end
|
99
|
+
|
86
100
|
|
87
101
|
def inputs
|
88
102
|
return @inputs if NamedArray === @inputs
|
@@ -49,7 +49,7 @@ class Step
|
|
49
49
|
end
|
50
50
|
end
|
51
51
|
|
52
|
-
def self.job_files_for_archive(files)
|
52
|
+
def self.job_files_for_archive(files, relocate = false)
|
53
53
|
job_files = Set.new
|
54
54
|
|
55
55
|
jobs = files.collect do |file|
|
@@ -71,10 +71,15 @@ class Step
|
|
71
71
|
seen = Set.new
|
72
72
|
while deps.any?
|
73
73
|
path = deps.shift
|
74
|
+
|
74
75
|
dep = Step.new path
|
75
76
|
seen << dep.path
|
76
|
-
|
77
|
-
|
77
|
+
|
78
|
+
dep.relocated = !!relocate
|
79
|
+
|
80
|
+
dep.load_dependencies_from_info
|
81
|
+
|
82
|
+
dep.dependencies.each do |dep|
|
78
83
|
next if seen.include? dep.path
|
79
84
|
deps << dep.path
|
80
85
|
rec_dependencies << dep.path
|
@@ -83,20 +88,21 @@ class Step
|
|
83
88
|
|
84
89
|
rec_dependencies.each do |path|
|
85
90
|
next unless File.exists?(path)
|
91
|
+
dep = Step.new path
|
92
|
+
job_files << dep.path
|
86
93
|
job_files << dep.files_dir if Dir.glob(dep.files_dir + '/*').any?
|
87
94
|
job_files << dep.info_file if File.exists?(dep.info_file)
|
88
|
-
job_files << path
|
89
95
|
end
|
90
96
|
end
|
91
97
|
|
92
98
|
job_files.to_a
|
93
99
|
end
|
94
100
|
|
95
|
-
def self.archive(files, target = nil)
|
101
|
+
def self.archive(files, target = nil, relocate = true)
|
96
102
|
target = self.path + '.tar.gz' if target.nil?
|
97
103
|
target = File.expand_path(target) if String === target
|
98
104
|
|
99
|
-
job_files = job_files_for_archive files
|
105
|
+
job_files = job_files_for_archive files, relocate
|
100
106
|
TmpFile.with_file do |tmpdir|
|
101
107
|
job_files.each do |file|
|
102
108
|
Step.link_job file, tmpdir
|
@@ -112,4 +118,117 @@ class Step
|
|
112
118
|
Log.debug "Archive finished at: #{target}"
|
113
119
|
end
|
114
120
|
end
|
121
|
+
|
122
|
+
def self.migrate(path, search_path, options = {})
|
123
|
+
|
124
|
+
resource=Rbbt
|
125
|
+
|
126
|
+
other_rsync_args = options[:rsync]
|
127
|
+
relocate = options[:relocate]
|
128
|
+
|
129
|
+
paths = if options[:source]
|
130
|
+
SSHClient.run(options[:source], <<-EOF).split("\n")
|
131
|
+
require 'rbbt-util'
|
132
|
+
require 'rbbt/workflow'
|
133
|
+
|
134
|
+
path = "#{path}"
|
135
|
+
relocate = #{ relocate.to_s }
|
136
|
+
if File.exists?(path)
|
137
|
+
path = #{resource.to_s}.identify(path)
|
138
|
+
else
|
139
|
+
path = Path.setup(path)
|
140
|
+
end
|
141
|
+
files = path.glob_all
|
142
|
+
if #{options[:recursive].to_s == 'true'}
|
143
|
+
files = Step.job_files_for_archive(files, relocate)
|
144
|
+
end
|
145
|
+
puts files * "\n"
|
146
|
+
EOF
|
147
|
+
else
|
148
|
+
if File.exists?(path)
|
149
|
+
path = resource.identify(path)
|
150
|
+
else
|
151
|
+
path = Path.setup(path)
|
152
|
+
end
|
153
|
+
files = path.glob_all
|
154
|
+
if options[:recursive]
|
155
|
+
files = Step.job_files_for_archive(files)
|
156
|
+
end
|
157
|
+
files
|
158
|
+
end
|
159
|
+
|
160
|
+
target = if options[:target]
|
161
|
+
target = SSHClient.run(options[:target], <<-EOF).split("\n").first
|
162
|
+
require 'rbbt-util'
|
163
|
+
path = "var/jobs"
|
164
|
+
resource = #{resource.to_s}
|
165
|
+
search_path = "#{search_path}"
|
166
|
+
puts resource[path].find(search_path)
|
167
|
+
EOF
|
168
|
+
else
|
169
|
+
resource['var/jobs'].find(search_path)
|
170
|
+
end
|
171
|
+
|
172
|
+
subpath_files = {}
|
173
|
+
paths.each do |source|
|
174
|
+
parts = source.split("/")
|
175
|
+
subpath = parts[0..-4] * "/"
|
176
|
+
source = parts[-3..-1] * "/"
|
177
|
+
subpath_files[subpath] ||= []
|
178
|
+
subpath_files[subpath] << source
|
179
|
+
end
|
180
|
+
|
181
|
+
subpath_files.each do |subpath, files|
|
182
|
+
if options[:target]
|
183
|
+
CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
|
184
|
+
else
|
185
|
+
Open.mkdir File.dirname(target)
|
186
|
+
end
|
187
|
+
|
188
|
+
if options[:source]
|
189
|
+
source = [options[:source], subpath] * ":"
|
190
|
+
else
|
191
|
+
source = subpath
|
192
|
+
end
|
193
|
+
target = [options[:target], target] * ":" if options[:target]
|
194
|
+
|
195
|
+
files_and_dirs = Set.new(files )
|
196
|
+
files.each do |file|
|
197
|
+
parts = file.split("/")[0..-2]
|
198
|
+
while parts.any?
|
199
|
+
files_and_dirs << parts * "/"
|
200
|
+
parts.pop
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
TmpFile.with_file(files_and_dirs.to_a * "\n") do |tmp_include_file|
|
205
|
+
test_str = options[:test] ? '-nv' : ''
|
206
|
+
|
207
|
+
includes_str = "--include-from='#{tmp_include_file}'"
|
208
|
+
cmd = "rsync -atAXmPL --progress #{test_str} --include-from='#{tmp_include_file}' --exclude='*' #{source}/ #{target}/ #{other_rsync_args}"
|
209
|
+
|
210
|
+
cmd << " && rm -Rf #{source}" if options[:delete]
|
211
|
+
|
212
|
+
if options[:print]
|
213
|
+
ppp Open.read(tmp_include_file)
|
214
|
+
puts cmd
|
215
|
+
else
|
216
|
+
CMD.cmd_log(cmd)
|
217
|
+
end
|
218
|
+
end
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
def self.purge(path, relocate = false)
|
223
|
+
path = [path] if String === path
|
224
|
+
job_files = job_files_for_archive path, relocate
|
225
|
+
|
226
|
+
job_files.each do |file|
|
227
|
+
begin
|
228
|
+
Open.rm_rf file if Open.exists?(file)
|
229
|
+
rescue
|
230
|
+
Log.warn "Could not erase '#{file}': #{$!.message}"
|
231
|
+
end
|
232
|
+
end
|
233
|
+
end
|
115
234
|
end
|
@@ -17,6 +17,7 @@ $ rbbt app start [options] <app_name>
|
|
17
17
|
-p--port* TCP port
|
18
18
|
-s--server* Server type: thin, webrick, unicorn, etc
|
19
19
|
-f--finder Start server with finder functionality
|
20
|
+
-fs--file_server Activate file serving for resources
|
20
21
|
-R--Rserve_session* Rserve session to use, otherwise start new one
|
21
22
|
-wd--workdir* Change the working directory of the workflow
|
22
23
|
--views* Directory with view templates
|
@@ -57,9 +58,11 @@ app_dir = app_dir[app]
|
|
57
58
|
server = options[:server] || 'puma'
|
58
59
|
Misc.in_dir(app_dir) do
|
59
60
|
require 'rack'
|
60
|
-
ENV["RBBT_FINDER"] = "true" if options
|
61
|
+
ENV["RBBT_FINDER"] = "true" if options[:finder]
|
61
62
|
ENV["RACK_ENV"] = options[:environment] if options.include?(:environment)
|
62
63
|
ENV["RBBT_VIEWS_DIR"] = options[:views] if options.include?(:views)
|
64
|
+
ENV["RBBT_REST_FILE_SERVER"] = "true" if options[:file_server]
|
65
|
+
|
63
66
|
|
64
67
|
if options[:stream]
|
65
68
|
#raise "No streaming available for any server other than puma" unless options[:server] =~ /^puma/
|
@@ -32,101 +32,6 @@ if options[:help]
|
|
32
32
|
exit 0
|
33
33
|
end
|
34
34
|
|
35
|
-
excludes = %w(.save .crap .source tmp filecache open-remote)
|
36
|
-
excludes += (options[:exclude] || "").split(/,\s*/)
|
37
|
-
excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
|
38
|
-
|
39
|
-
test_str = options[:test] ? '-nv' : ''
|
40
|
-
|
41
35
|
path, search_path, _sep, *other = ARGV
|
42
36
|
|
43
|
-
|
44
|
-
|
45
|
-
paths = if options[:source]
|
46
|
-
SSHClient.run(options[:source], <<-EOF).split("\n")
|
47
|
-
require 'rbbt-util'
|
48
|
-
require 'rbbt/workflow'
|
49
|
-
|
50
|
-
path = "#{path}"
|
51
|
-
if File.exists?(path)
|
52
|
-
path = #{resource.to_s}.identify(path)
|
53
|
-
else
|
54
|
-
path = Path.setup(path)
|
55
|
-
end
|
56
|
-
files = path.glob_all
|
57
|
-
if #{options[:recursive].to_s == 'true'}
|
58
|
-
files = Step.job_files_for_archive(files)
|
59
|
-
end
|
60
|
-
puts files * "\n"
|
61
|
-
EOF
|
62
|
-
else
|
63
|
-
if File.exists?(path)
|
64
|
-
path = resource.identify(path)
|
65
|
-
else
|
66
|
-
path = Path.setup(path)
|
67
|
-
end
|
68
|
-
files = path.glob_all
|
69
|
-
if options[:recursive]
|
70
|
-
files = Step.job_files_for_archive(files)
|
71
|
-
end
|
72
|
-
files
|
73
|
-
end
|
74
|
-
|
75
|
-
target = if options[:target]
|
76
|
-
target = SSHClient.run(options[:target], <<-EOF).split("\n").first
|
77
|
-
require 'rbbt-util'
|
78
|
-
path = "var/jobs"
|
79
|
-
resource = #{resource.to_s}
|
80
|
-
search_path = "#{search_path}"
|
81
|
-
puts resource[path].find(search_path)
|
82
|
-
EOF
|
83
|
-
else
|
84
|
-
resource['var/jobs'].find(search_path)
|
85
|
-
end
|
86
|
-
|
87
|
-
subpath_files = {}
|
88
|
-
paths.each do |source|
|
89
|
-
parts = source.split("/")
|
90
|
-
subpath = parts[0..-4] * "/"
|
91
|
-
source = parts[-3..-1] * "/"
|
92
|
-
subpath_files[subpath] ||= []
|
93
|
-
subpath_files[subpath] << source
|
94
|
-
end
|
95
|
-
|
96
|
-
subpath_files.each do |subpath, files|
|
97
|
-
if options[:target]
|
98
|
-
CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
|
99
|
-
else
|
100
|
-
Open.mkdir File.dirname(target)
|
101
|
-
end
|
102
|
-
|
103
|
-
if options[:source]
|
104
|
-
source = [options[:source], subpath] * ":"
|
105
|
-
else
|
106
|
-
source = subpath
|
107
|
-
end
|
108
|
-
target = [options[:target], target] * ":" if options[:target]
|
109
|
-
|
110
|
-
files_and_dirs = Set.new(files )
|
111
|
-
files.each do |file|
|
112
|
-
parts = file.split("/")[0..-2]
|
113
|
-
while parts.any?
|
114
|
-
files_and_dirs << parts * "/"
|
115
|
-
parts.pop
|
116
|
-
end
|
117
|
-
end
|
118
|
-
|
119
|
-
TmpFile.with_file(files_and_dirs.to_a * "\n") do |tmp_include_file|
|
120
|
-
includes_str = "--include-from='#{tmp_include_file}'"
|
121
|
-
cmd = "rsync -atAXmPL --progress #{test_str} --include-from='#{tmp_include_file}' --exclude='*' #{source}/ #{target}/ #{other * " "}"
|
122
|
-
|
123
|
-
cmd << " && rm -Rf #{source}" if options[:delete]
|
124
|
-
|
125
|
-
if options[:print]
|
126
|
-
ppp Open.read(tmp_include_file)
|
127
|
-
puts cmd
|
128
|
-
else
|
129
|
-
CMD.cmd_log(cmd)
|
130
|
-
end
|
131
|
-
end
|
132
|
-
end
|
37
|
+
Step.migrate(path, search_path, options)
|
@@ -0,0 +1,36 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rbbt-util'
|
4
|
+
require 'rbbt/util/simpleopt'
|
5
|
+
require 'rbbt/workflow'
|
6
|
+
require 'rbbt/workflow/remote/ssh/get'
|
7
|
+
|
8
|
+
$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
|
9
|
+
|
10
|
+
options = SOPT.setup <<EOF
|
11
|
+
|
12
|
+
Remove a job and its dependencies
|
13
|
+
|
14
|
+
$ rbbt purge [options] <job_path>
|
15
|
+
|
16
|
+
You can specify extra rsync options after --
|
17
|
+
|
18
|
+
-h--help Print this help
|
19
|
+
-t--test Do a verbose dry run
|
20
|
+
-r--relocate Include relocated dependencies
|
21
|
+
|
22
|
+
EOF
|
23
|
+
if options[:help]
|
24
|
+
if defined? rbbt_usage
|
25
|
+
rbbt_usage
|
26
|
+
else
|
27
|
+
puts SOPT.doc
|
28
|
+
end
|
29
|
+
exit 0
|
30
|
+
end
|
31
|
+
|
32
|
+
path, search_path, _sep, *other = ARGV
|
33
|
+
|
34
|
+
raise ParameterException, "No path given" if path.nil?
|
35
|
+
|
36
|
+
Step.purge(path, options[:relocate])
|
@@ -23,6 +23,7 @@ $ rbbt workflow server [options] <Workflow>
|
|
23
23
|
-R--requires* Require a list of files
|
24
24
|
--views* Directory with view templates
|
25
25
|
--stream Activate streaming of workflow tasks
|
26
|
+
-fs--file_server Activate file serving for resources
|
26
27
|
--export_all Export all workflow tasks (use with caution!)
|
27
28
|
--export* Export workflow tasks (asynchronous)
|
28
29
|
--export_asynchronous* Export workflow tasks as asynchronous
|
@@ -81,9 +82,10 @@ TmpFile.with_file do |app_dir|
|
|
81
82
|
Open.write(app_dir.etc.requires.find, requires.split(/,\s*/)*"\n") if requires and not requires.empty?
|
82
83
|
|
83
84
|
require 'rack'
|
84
|
-
ENV["RBBT_FINDER"] = "true" if options
|
85
|
+
ENV["RBBT_FINDER"] = "true" if options[:finder]
|
85
86
|
ENV["RACK_ENV"] = options[:environment] if options.include?(:environment)
|
86
87
|
ENV["RBBT_VIEWS_DIR"] = options[:views] if options.include?(:views)
|
88
|
+
ENV["RBBT_REST_FILE_SERVER"] = "true" if options[:file_server]
|
87
89
|
|
88
90
|
if options[:export_all]
|
89
91
|
ENV["RBBT_WORKFLOW_EXPORT_ALL"] = 'true'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.26.
|
4
|
+
version: 5.26.79
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-10-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -367,6 +367,7 @@ files:
|
|
367
367
|
- share/rbbt_commands/log
|
368
368
|
- share/rbbt_commands/migrate
|
369
369
|
- share/rbbt_commands/migrate_job
|
370
|
+
- share/rbbt_commands/purge_job
|
370
371
|
- share/rbbt_commands/resource/exists
|
371
372
|
- share/rbbt_commands/resource/find
|
372
373
|
- share/rbbt_commands/resource/get
|