rbbt-util 5.26.78 → 5.26.79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/etc/app.d/base.rb +5 -1
- data/lib/rbbt/resource/path.rb +1 -0
- data/lib/rbbt/resource/util.rb +1 -1
- data/lib/rbbt/resource.rb +1 -0
- data/lib/rbbt/workflow/step.rb +14 -0
- data/lib/rbbt/workflow/util/archive.rb +125 -6
- data/share/rbbt_commands/app/start +4 -1
- data/share/rbbt_commands/migrate_job +1 -96
- data/share/rbbt_commands/purge_job +36 -0
- data/share/rbbt_commands/workflow/server +3 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d7b3f571fcdf2bf96368174818464415b1282a2d08e71fda696d325b1d2a84f4
|
4
|
+
data.tar.gz: f0403774806dc37731e0cfe77bdbdc7908bc6aee9015b657871c786bbd1f849c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '0988f55f748f4917507a819e0bc367bb307893cdc18654bd3af040f4a4f1ade4b64656802c388bfeb0eb9e297e20e1392db869ff4fe9d81871eaccfbf2893e3a'
|
7
|
+
data.tar.gz: e0e865fc6e92866da62eef19bdbdb7fe3dd4036e6fc3f730c01f6823dd5d970477c568d7451bdeab5be76ace260552a19be2d04661a33da79630750fd4fb3143
|
data/etc/app.d/base.rb
CHANGED
@@ -3,13 +3,17 @@ register Sinatra::MultiRoute
|
|
3
3
|
|
4
4
|
register Sinatra::RbbtRESTMain
|
5
5
|
register Sinatra::RbbtRESTEntity
|
6
|
-
register Sinatra::RbbtRESTFileServer # Remove to prevent serving files
|
7
6
|
register Sinatra::RbbtRESTKnowledgeBase
|
8
7
|
register Sinatra::RbbtRESTWorkflow
|
9
8
|
|
10
9
|
helpers Sinatra::RbbtToolHelper
|
11
10
|
helpers Sinatra::RbbtMiscHelpers
|
12
11
|
|
12
|
+
if ENV["RBBT_REST_FILE_SERVER"] == 'true'
|
13
|
+
Log.high "Activating File Server"
|
14
|
+
register Sinatra::RbbtRESTFileServer
|
15
|
+
end
|
16
|
+
|
13
17
|
#{{{ SESSIONS
|
14
18
|
|
15
19
|
use Rack::Session::Cookie, :key => 'rack.session',
|
data/lib/rbbt/resource/path.rb
CHANGED
data/lib/rbbt/resource/util.rb
CHANGED
@@ -25,7 +25,7 @@ module Path
|
|
25
25
|
SLASH = "/"[0]
|
26
26
|
DOT = "."[0]
|
27
27
|
def located?
|
28
|
-
self.byte(0) == SLASH
|
28
|
+
self.byte(0) == SLASH || (self.byte(0) == DOT && self.byte(1) == SLASH) || (resource != Rbbt && (Open.remote?(self) || Open.ssh?(self)))
|
29
29
|
end
|
30
30
|
end
|
31
31
|
|
data/lib/rbbt/resource.rb
CHANGED
data/lib/rbbt/workflow/step.rb
CHANGED
@@ -83,6 +83,20 @@ class Step
|
|
83
83
|
end
|
84
84
|
end
|
85
85
|
|
86
|
+
def load_dependencies_from_info
|
87
|
+
@dependencies = (self.info[:dependencies] || []).collect do |task,name,dep_path|
|
88
|
+
if Open.exists?(dep_path) || Open.exists?(dep_path + '.info')
|
89
|
+
Workflow._load_step dep_path
|
90
|
+
else
|
91
|
+
next if FalseClass === relocated
|
92
|
+
new_path = Workflow.relocate(path, dep_path)
|
93
|
+
relocated = true if Open.exists?(new_path) || Open.exists?(new_path + '.info')
|
94
|
+
Workflow._load_step new_path
|
95
|
+
end
|
96
|
+
end.compact
|
97
|
+
@relocated = relocated
|
98
|
+
end
|
99
|
+
|
86
100
|
|
87
101
|
def inputs
|
88
102
|
return @inputs if NamedArray === @inputs
|
@@ -49,7 +49,7 @@ class Step
|
|
49
49
|
end
|
50
50
|
end
|
51
51
|
|
52
|
-
def self.job_files_for_archive(files)
|
52
|
+
def self.job_files_for_archive(files, relocate = false)
|
53
53
|
job_files = Set.new
|
54
54
|
|
55
55
|
jobs = files.collect do |file|
|
@@ -71,10 +71,15 @@ class Step
|
|
71
71
|
seen = Set.new
|
72
72
|
while deps.any?
|
73
73
|
path = deps.shift
|
74
|
+
|
74
75
|
dep = Step.new path
|
75
76
|
seen << dep.path
|
76
|
-
|
77
|
-
|
77
|
+
|
78
|
+
dep.relocated = !!relocate
|
79
|
+
|
80
|
+
dep.load_dependencies_from_info
|
81
|
+
|
82
|
+
dep.dependencies.each do |dep|
|
78
83
|
next if seen.include? dep.path
|
79
84
|
deps << dep.path
|
80
85
|
rec_dependencies << dep.path
|
@@ -83,20 +88,21 @@ class Step
|
|
83
88
|
|
84
89
|
rec_dependencies.each do |path|
|
85
90
|
next unless File.exists?(path)
|
91
|
+
dep = Step.new path
|
92
|
+
job_files << dep.path
|
86
93
|
job_files << dep.files_dir if Dir.glob(dep.files_dir + '/*').any?
|
87
94
|
job_files << dep.info_file if File.exists?(dep.info_file)
|
88
|
-
job_files << path
|
89
95
|
end
|
90
96
|
end
|
91
97
|
|
92
98
|
job_files.to_a
|
93
99
|
end
|
94
100
|
|
95
|
-
def self.archive(files, target = nil)
|
101
|
+
def self.archive(files, target = nil, relocate = true)
|
96
102
|
target = self.path + '.tar.gz' if target.nil?
|
97
103
|
target = File.expand_path(target) if String === target
|
98
104
|
|
99
|
-
job_files = job_files_for_archive files
|
105
|
+
job_files = job_files_for_archive files, relocate
|
100
106
|
TmpFile.with_file do |tmpdir|
|
101
107
|
job_files.each do |file|
|
102
108
|
Step.link_job file, tmpdir
|
@@ -112,4 +118,117 @@ class Step
|
|
112
118
|
Log.debug "Archive finished at: #{target}"
|
113
119
|
end
|
114
120
|
end
|
121
|
+
|
122
|
+
def self.migrate(path, search_path, options = {})
|
123
|
+
|
124
|
+
resource=Rbbt
|
125
|
+
|
126
|
+
other_rsync_args = options[:rsync]
|
127
|
+
relocate = options[:relocate]
|
128
|
+
|
129
|
+
paths = if options[:source]
|
130
|
+
SSHClient.run(options[:source], <<-EOF).split("\n")
|
131
|
+
require 'rbbt-util'
|
132
|
+
require 'rbbt/workflow'
|
133
|
+
|
134
|
+
path = "#{path}"
|
135
|
+
relocate = #{ relocate.to_s }
|
136
|
+
if File.exists?(path)
|
137
|
+
path = #{resource.to_s}.identify(path)
|
138
|
+
else
|
139
|
+
path = Path.setup(path)
|
140
|
+
end
|
141
|
+
files = path.glob_all
|
142
|
+
if #{options[:recursive].to_s == 'true'}
|
143
|
+
files = Step.job_files_for_archive(files, relocate)
|
144
|
+
end
|
145
|
+
puts files * "\n"
|
146
|
+
EOF
|
147
|
+
else
|
148
|
+
if File.exists?(path)
|
149
|
+
path = resource.identify(path)
|
150
|
+
else
|
151
|
+
path = Path.setup(path)
|
152
|
+
end
|
153
|
+
files = path.glob_all
|
154
|
+
if options[:recursive]
|
155
|
+
files = Step.job_files_for_archive(files)
|
156
|
+
end
|
157
|
+
files
|
158
|
+
end
|
159
|
+
|
160
|
+
target = if options[:target]
|
161
|
+
target = SSHClient.run(options[:target], <<-EOF).split("\n").first
|
162
|
+
require 'rbbt-util'
|
163
|
+
path = "var/jobs"
|
164
|
+
resource = #{resource.to_s}
|
165
|
+
search_path = "#{search_path}"
|
166
|
+
puts resource[path].find(search_path)
|
167
|
+
EOF
|
168
|
+
else
|
169
|
+
resource['var/jobs'].find(search_path)
|
170
|
+
end
|
171
|
+
|
172
|
+
subpath_files = {}
|
173
|
+
paths.each do |source|
|
174
|
+
parts = source.split("/")
|
175
|
+
subpath = parts[0..-4] * "/"
|
176
|
+
source = parts[-3..-1] * "/"
|
177
|
+
subpath_files[subpath] ||= []
|
178
|
+
subpath_files[subpath] << source
|
179
|
+
end
|
180
|
+
|
181
|
+
subpath_files.each do |subpath, files|
|
182
|
+
if options[:target]
|
183
|
+
CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
|
184
|
+
else
|
185
|
+
Open.mkdir File.dirname(target)
|
186
|
+
end
|
187
|
+
|
188
|
+
if options[:source]
|
189
|
+
source = [options[:source], subpath] * ":"
|
190
|
+
else
|
191
|
+
source = subpath
|
192
|
+
end
|
193
|
+
target = [options[:target], target] * ":" if options[:target]
|
194
|
+
|
195
|
+
files_and_dirs = Set.new(files )
|
196
|
+
files.each do |file|
|
197
|
+
parts = file.split("/")[0..-2]
|
198
|
+
while parts.any?
|
199
|
+
files_and_dirs << parts * "/"
|
200
|
+
parts.pop
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
TmpFile.with_file(files_and_dirs.to_a * "\n") do |tmp_include_file|
|
205
|
+
test_str = options[:test] ? '-nv' : ''
|
206
|
+
|
207
|
+
includes_str = "--include-from='#{tmp_include_file}'"
|
208
|
+
cmd = "rsync -atAXmPL --progress #{test_str} --include-from='#{tmp_include_file}' --exclude='*' #{source}/ #{target}/ #{other_rsync_args}"
|
209
|
+
|
210
|
+
cmd << " && rm -Rf #{source}" if options[:delete]
|
211
|
+
|
212
|
+
if options[:print]
|
213
|
+
ppp Open.read(tmp_include_file)
|
214
|
+
puts cmd
|
215
|
+
else
|
216
|
+
CMD.cmd_log(cmd)
|
217
|
+
end
|
218
|
+
end
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
def self.purge(path, relocate = false)
|
223
|
+
path = [path] if String === path
|
224
|
+
job_files = job_files_for_archive path, relocate
|
225
|
+
|
226
|
+
job_files.each do |file|
|
227
|
+
begin
|
228
|
+
Open.rm_rf file if Open.exists?(file)
|
229
|
+
rescue
|
230
|
+
Log.warn "Could not erase '#{file}': #{$!.message}"
|
231
|
+
end
|
232
|
+
end
|
233
|
+
end
|
115
234
|
end
|
@@ -17,6 +17,7 @@ $ rbbt app start [options] <app_name>
|
|
17
17
|
-p--port* TCP port
|
18
18
|
-s--server* Server type: thin, webrick, unicorn, etc
|
19
19
|
-f--finder Start server with finder functionality
|
20
|
+
-fs--file_server Activate file serving for resources
|
20
21
|
-R--Rserve_session* Rserve session to use, otherwise start new one
|
21
22
|
-wd--workdir* Change the working directory of the workflow
|
22
23
|
--views* Directory with view templates
|
@@ -57,9 +58,11 @@ app_dir = app_dir[app]
|
|
57
58
|
server = options[:server] || 'puma'
|
58
59
|
Misc.in_dir(app_dir) do
|
59
60
|
require 'rack'
|
60
|
-
ENV["RBBT_FINDER"] = "true" if options
|
61
|
+
ENV["RBBT_FINDER"] = "true" if options[:finder]
|
61
62
|
ENV["RACK_ENV"] = options[:environment] if options.include?(:environment)
|
62
63
|
ENV["RBBT_VIEWS_DIR"] = options[:views] if options.include?(:views)
|
64
|
+
ENV["RBBT_REST_FILE_SERVER"] = "true" if options[:file_server]
|
65
|
+
|
63
66
|
|
64
67
|
if options[:stream]
|
65
68
|
#raise "No streaming available for any server other than puma" unless options[:server] =~ /^puma/
|
@@ -32,101 +32,6 @@ if options[:help]
|
|
32
32
|
exit 0
|
33
33
|
end
|
34
34
|
|
35
|
-
excludes = %w(.save .crap .source tmp filecache open-remote)
|
36
|
-
excludes += (options[:exclude] || "").split(/,\s*/)
|
37
|
-
excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
|
38
|
-
|
39
|
-
test_str = options[:test] ? '-nv' : ''
|
40
|
-
|
41
35
|
path, search_path, _sep, *other = ARGV
|
42
36
|
|
43
|
-
|
44
|
-
|
45
|
-
paths = if options[:source]
|
46
|
-
SSHClient.run(options[:source], <<-EOF).split("\n")
|
47
|
-
require 'rbbt-util'
|
48
|
-
require 'rbbt/workflow'
|
49
|
-
|
50
|
-
path = "#{path}"
|
51
|
-
if File.exists?(path)
|
52
|
-
path = #{resource.to_s}.identify(path)
|
53
|
-
else
|
54
|
-
path = Path.setup(path)
|
55
|
-
end
|
56
|
-
files = path.glob_all
|
57
|
-
if #{options[:recursive].to_s == 'true'}
|
58
|
-
files = Step.job_files_for_archive(files)
|
59
|
-
end
|
60
|
-
puts files * "\n"
|
61
|
-
EOF
|
62
|
-
else
|
63
|
-
if File.exists?(path)
|
64
|
-
path = resource.identify(path)
|
65
|
-
else
|
66
|
-
path = Path.setup(path)
|
67
|
-
end
|
68
|
-
files = path.glob_all
|
69
|
-
if options[:recursive]
|
70
|
-
files = Step.job_files_for_archive(files)
|
71
|
-
end
|
72
|
-
files
|
73
|
-
end
|
74
|
-
|
75
|
-
target = if options[:target]
|
76
|
-
target = SSHClient.run(options[:target], <<-EOF).split("\n").first
|
77
|
-
require 'rbbt-util'
|
78
|
-
path = "var/jobs"
|
79
|
-
resource = #{resource.to_s}
|
80
|
-
search_path = "#{search_path}"
|
81
|
-
puts resource[path].find(search_path)
|
82
|
-
EOF
|
83
|
-
else
|
84
|
-
resource['var/jobs'].find(search_path)
|
85
|
-
end
|
86
|
-
|
87
|
-
subpath_files = {}
|
88
|
-
paths.each do |source|
|
89
|
-
parts = source.split("/")
|
90
|
-
subpath = parts[0..-4] * "/"
|
91
|
-
source = parts[-3..-1] * "/"
|
92
|
-
subpath_files[subpath] ||= []
|
93
|
-
subpath_files[subpath] << source
|
94
|
-
end
|
95
|
-
|
96
|
-
subpath_files.each do |subpath, files|
|
97
|
-
if options[:target]
|
98
|
-
CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
|
99
|
-
else
|
100
|
-
Open.mkdir File.dirname(target)
|
101
|
-
end
|
102
|
-
|
103
|
-
if options[:source]
|
104
|
-
source = [options[:source], subpath] * ":"
|
105
|
-
else
|
106
|
-
source = subpath
|
107
|
-
end
|
108
|
-
target = [options[:target], target] * ":" if options[:target]
|
109
|
-
|
110
|
-
files_and_dirs = Set.new(files )
|
111
|
-
files.each do |file|
|
112
|
-
parts = file.split("/")[0..-2]
|
113
|
-
while parts.any?
|
114
|
-
files_and_dirs << parts * "/"
|
115
|
-
parts.pop
|
116
|
-
end
|
117
|
-
end
|
118
|
-
|
119
|
-
TmpFile.with_file(files_and_dirs.to_a * "\n") do |tmp_include_file|
|
120
|
-
includes_str = "--include-from='#{tmp_include_file}'"
|
121
|
-
cmd = "rsync -atAXmPL --progress #{test_str} --include-from='#{tmp_include_file}' --exclude='*' #{source}/ #{target}/ #{other * " "}"
|
122
|
-
|
123
|
-
cmd << " && rm -Rf #{source}" if options[:delete]
|
124
|
-
|
125
|
-
if options[:print]
|
126
|
-
ppp Open.read(tmp_include_file)
|
127
|
-
puts cmd
|
128
|
-
else
|
129
|
-
CMD.cmd_log(cmd)
|
130
|
-
end
|
131
|
-
end
|
132
|
-
end
|
37
|
+
Step.migrate(path, search_path, options)
|
@@ -0,0 +1,36 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rbbt-util'
|
4
|
+
require 'rbbt/util/simpleopt'
|
5
|
+
require 'rbbt/workflow'
|
6
|
+
require 'rbbt/workflow/remote/ssh/get'
|
7
|
+
|
8
|
+
$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
|
9
|
+
|
10
|
+
options = SOPT.setup <<EOF
|
11
|
+
|
12
|
+
Remove a job and its dependencies
|
13
|
+
|
14
|
+
$ rbbt purge [options] <job_path>
|
15
|
+
|
16
|
+
You can specify extra rsync options after --
|
17
|
+
|
18
|
+
-h--help Print this help
|
19
|
+
-t--test Do a verbose dry run
|
20
|
+
-r--relocate Include relocated dependencies
|
21
|
+
|
22
|
+
EOF
|
23
|
+
if options[:help]
|
24
|
+
if defined? rbbt_usage
|
25
|
+
rbbt_usage
|
26
|
+
else
|
27
|
+
puts SOPT.doc
|
28
|
+
end
|
29
|
+
exit 0
|
30
|
+
end
|
31
|
+
|
32
|
+
path, search_path, _sep, *other = ARGV
|
33
|
+
|
34
|
+
raise ParameterException, "No path given" if path.nil?
|
35
|
+
|
36
|
+
Step.purge(path, options[:relocate])
|
@@ -23,6 +23,7 @@ $ rbbt workflow server [options] <Workflow>
|
|
23
23
|
-R--requires* Require a list of files
|
24
24
|
--views* Directory with view templates
|
25
25
|
--stream Activate streaming of workflow tasks
|
26
|
+
-fs--file_server Activate file serving for resources
|
26
27
|
--export_all Export all workflow tasks (use with caution!)
|
27
28
|
--export* Export workflow tasks (asynchronous)
|
28
29
|
--export_asynchronous* Export workflow tasks as asynchronous
|
@@ -81,9 +82,10 @@ TmpFile.with_file do |app_dir|
|
|
81
82
|
Open.write(app_dir.etc.requires.find, requires.split(/,\s*/)*"\n") if requires and not requires.empty?
|
82
83
|
|
83
84
|
require 'rack'
|
84
|
-
ENV["RBBT_FINDER"] = "true" if options
|
85
|
+
ENV["RBBT_FINDER"] = "true" if options[:finder]
|
85
86
|
ENV["RACK_ENV"] = options[:environment] if options.include?(:environment)
|
86
87
|
ENV["RBBT_VIEWS_DIR"] = options[:views] if options.include?(:views)
|
88
|
+
ENV["RBBT_REST_FILE_SERVER"] = "true" if options[:file_server]
|
87
89
|
|
88
90
|
if options[:export_all]
|
89
91
|
ENV["RBBT_WORKFLOW_EXPORT_ALL"] = 'true'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.26.
|
4
|
+
version: 5.26.79
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-10-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -367,6 +367,7 @@ files:
|
|
367
367
|
- share/rbbt_commands/log
|
368
368
|
- share/rbbt_commands/migrate
|
369
369
|
- share/rbbt_commands/migrate_job
|
370
|
+
- share/rbbt_commands/purge_job
|
370
371
|
- share/rbbt_commands/resource/exists
|
371
372
|
- share/rbbt_commands/resource/find
|
372
373
|
- share/rbbt_commands/resource/get
|