rbbt-util 5.26.78 → 5.26.79

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fa385d029073f92c7d003f8dd139ffd76f256cf50e8b6f9eb48146b8624c1d0f
4
- data.tar.gz: 1e9d23cd39e36dcebf0d2e61f269593f031cb7a8c8b4ff900993aa10fd7b11a6
3
+ metadata.gz: d7b3f571fcdf2bf96368174818464415b1282a2d08e71fda696d325b1d2a84f4
4
+ data.tar.gz: f0403774806dc37731e0cfe77bdbdc7908bc6aee9015b657871c786bbd1f849c
5
5
  SHA512:
6
- metadata.gz: 3f7f0f8aef95c5f79832f18c741f6751e24ff78aa59e739fa44b38d5ae88c5be7f77ae1c293fb07c025e6ed9ed6030354b8b279ed379a01bf970a0efb9d5bd99
7
- data.tar.gz: 02070acbc4a9ab3925f0bbf85bc2b07069241d46e8ffdf72085e09bd952b3e24d73eceed255c267d2b1de9ed63fb214142413a9d3e0badb3f19b409c5880a4b6
6
+ metadata.gz: '0988f55f748f4917507a819e0bc367bb307893cdc18654bd3af040f4a4f1ade4b64656802c388bfeb0eb9e297e20e1392db869ff4fe9d81871eaccfbf2893e3a'
7
+ data.tar.gz: e0e865fc6e92866da62eef19bdbdb7fe3dd4036e6fc3f730c01f6823dd5d970477c568d7451bdeab5be76ace260552a19be2d04661a33da79630750fd4fb3143
data/etc/app.d/base.rb CHANGED
@@ -3,13 +3,17 @@ register Sinatra::MultiRoute
3
3
 
4
4
  register Sinatra::RbbtRESTMain
5
5
  register Sinatra::RbbtRESTEntity
6
- register Sinatra::RbbtRESTFileServer # Remove to prevent serving files
7
6
  register Sinatra::RbbtRESTKnowledgeBase
8
7
  register Sinatra::RbbtRESTWorkflow
9
8
 
10
9
  helpers Sinatra::RbbtToolHelper
11
10
  helpers Sinatra::RbbtMiscHelpers
12
11
 
12
+ if ENV["RBBT_REST_FILE_SERVER"] == 'true'
13
+ Log.high "Activating File Server"
14
+ register Sinatra::RbbtRESTFileServer
15
+ end
16
+
13
17
  #{{{ SESSIONS
14
18
 
15
19
  use Rack::Session::Cookie, :key => 'rack.session',
@@ -236,6 +236,7 @@ module Path
236
236
  sub('{BASENAME}', File.basename(self)).
237
237
  sub('{PATH}', self).
238
238
  sub('{LIBDIR}', libdir).
239
+ sub('{RESOURCE}', resource.to_s).
239
240
  sub('{REMOVE}/', '').
240
241
  sub('{REMOVE}', '')
241
242
 
@@ -25,7 +25,7 @@ module Path
25
25
  SLASH = "/"[0]
26
26
  DOT = "."[0]
27
27
  def located?
28
- self.byte(0) == SLASH or (self.byte(0) == DOT and self.byte(1) == SLASH)
28
+ self.byte(0) == SLASH || (self.byte(0) == DOT && self.byte(1) == SLASH) || (resource != Rbbt && (Open.remote?(self) || Open.ssh?(self)))
29
29
  end
30
30
  end
31
31
 
data/lib/rbbt/resource.rb CHANGED
@@ -297,6 +297,7 @@ url='#{url}'
297
297
  end
298
298
  end
299
299
  end
300
+ nil
300
301
  end
301
302
  end
302
303
 
@@ -83,6 +83,20 @@ class Step
83
83
  end
84
84
  end
85
85
 
86
+ def load_dependencies_from_info
87
+ @dependencies = (self.info[:dependencies] || []).collect do |task,name,dep_path|
88
+ if Open.exists?(dep_path) || Open.exists?(dep_path + '.info')
89
+ Workflow._load_step dep_path
90
+ else
91
+ next if FalseClass === relocated
92
+ new_path = Workflow.relocate(path, dep_path)
93
+ relocated = true if Open.exists?(new_path) || Open.exists?(new_path + '.info')
94
+ Workflow._load_step new_path
95
+ end
96
+ end.compact
97
+ @relocated = relocated
98
+ end
99
+
86
100
 
87
101
  def inputs
88
102
  return @inputs if NamedArray === @inputs
@@ -49,7 +49,7 @@ class Step
49
49
  end
50
50
  end
51
51
 
52
- def self.job_files_for_archive(files)
52
+ def self.job_files_for_archive(files, relocate = false)
53
53
  job_files = Set.new
54
54
 
55
55
  jobs = files.collect do |file|
@@ -71,10 +71,15 @@ class Step
71
71
  seen = Set.new
72
72
  while deps.any?
73
73
  path = deps.shift
74
+
74
75
  dep = Step.new path
75
76
  seen << dep.path
76
- dep.info[:dependencies].each do |task, name, path|
77
- dep = Step.new path
77
+
78
+ dep.relocated = !!relocate
79
+
80
+ dep.load_dependencies_from_info
81
+
82
+ dep.dependencies.each do |dep|
78
83
  next if seen.include? dep.path
79
84
  deps << dep.path
80
85
  rec_dependencies << dep.path
@@ -83,20 +88,21 @@ class Step
83
88
 
84
89
  rec_dependencies.each do |path|
85
90
  next unless File.exists?(path)
91
+ dep = Step.new path
92
+ job_files << dep.path
86
93
  job_files << dep.files_dir if Dir.glob(dep.files_dir + '/*').any?
87
94
  job_files << dep.info_file if File.exists?(dep.info_file)
88
- job_files << path
89
95
  end
90
96
  end
91
97
 
92
98
  job_files.to_a
93
99
  end
94
100
 
95
- def self.archive(files, target = nil)
101
+ def self.archive(files, target = nil, relocate = true)
96
102
  target = self.path + '.tar.gz' if target.nil?
97
103
  target = File.expand_path(target) if String === target
98
104
 
99
- job_files = job_files_for_archive files
105
+ job_files = job_files_for_archive files, relocate
100
106
  TmpFile.with_file do |tmpdir|
101
107
  job_files.each do |file|
102
108
  Step.link_job file, tmpdir
@@ -112,4 +118,117 @@ class Step
112
118
  Log.debug "Archive finished at: #{target}"
113
119
  end
114
120
  end
121
+
122
+ def self.migrate(path, search_path, options = {})
123
+
124
+ resource=Rbbt
125
+
126
+ other_rsync_args = options[:rsync]
127
+ relocate = options[:relocate]
128
+
129
+ paths = if options[:source]
130
+ SSHClient.run(options[:source], <<-EOF).split("\n")
131
+ require 'rbbt-util'
132
+ require 'rbbt/workflow'
133
+
134
+ path = "#{path}"
135
+ relocate = #{ relocate.to_s }
136
+ if File.exists?(path)
137
+ path = #{resource.to_s}.identify(path)
138
+ else
139
+ path = Path.setup(path)
140
+ end
141
+ files = path.glob_all
142
+ if #{options[:recursive].to_s == 'true'}
143
+ files = Step.job_files_for_archive(files, relocate)
144
+ end
145
+ puts files * "\n"
146
+ EOF
147
+ else
148
+ if File.exists?(path)
149
+ path = resource.identify(path)
150
+ else
151
+ path = Path.setup(path)
152
+ end
153
+ files = path.glob_all
154
+ if options[:recursive]
155
+ files = Step.job_files_for_archive(files)
156
+ end
157
+ files
158
+ end
159
+
160
+ target = if options[:target]
161
+ target = SSHClient.run(options[:target], <<-EOF).split("\n").first
162
+ require 'rbbt-util'
163
+ path = "var/jobs"
164
+ resource = #{resource.to_s}
165
+ search_path = "#{search_path}"
166
+ puts resource[path].find(search_path)
167
+ EOF
168
+ else
169
+ resource['var/jobs'].find(search_path)
170
+ end
171
+
172
+ subpath_files = {}
173
+ paths.each do |source|
174
+ parts = source.split("/")
175
+ subpath = parts[0..-4] * "/"
176
+ source = parts[-3..-1] * "/"
177
+ subpath_files[subpath] ||= []
178
+ subpath_files[subpath] << source
179
+ end
180
+
181
+ subpath_files.each do |subpath, files|
182
+ if options[:target]
183
+ CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
184
+ else
185
+ Open.mkdir File.dirname(target)
186
+ end
187
+
188
+ if options[:source]
189
+ source = [options[:source], subpath] * ":"
190
+ else
191
+ source = subpath
192
+ end
193
+ target = [options[:target], target] * ":" if options[:target]
194
+
195
+ files_and_dirs = Set.new(files )
196
+ files.each do |file|
197
+ parts = file.split("/")[0..-2]
198
+ while parts.any?
199
+ files_and_dirs << parts * "/"
200
+ parts.pop
201
+ end
202
+ end
203
+
204
+ TmpFile.with_file(files_and_dirs.to_a * "\n") do |tmp_include_file|
205
+ test_str = options[:test] ? '-nv' : ''
206
+
207
+ includes_str = "--include-from='#{tmp_include_file}'"
208
+ cmd = "rsync -atAXmPL --progress #{test_str} --include-from='#{tmp_include_file}' --exclude='*' #{source}/ #{target}/ #{other_rsync_args}"
209
+
210
+ cmd << " && rm -Rf #{source}" if options[:delete]
211
+
212
+ if options[:print]
213
+ ppp Open.read(tmp_include_file)
214
+ puts cmd
215
+ else
216
+ CMD.cmd_log(cmd)
217
+ end
218
+ end
219
+ end
220
+ end
221
+
222
+ def self.purge(path, relocate = false)
223
+ path = [path] if String === path
224
+ job_files = job_files_for_archive path, relocate
225
+
226
+ job_files.each do |file|
227
+ begin
228
+ Open.rm_rf file if Open.exists?(file)
229
+ rescue
230
+ Log.warn "Could not erase '#{file}': #{$!.message}"
231
+ end
232
+ end
233
+ end
115
234
  end
@@ -17,6 +17,7 @@ $ rbbt app start [options] <app_name>
17
17
  -p--port* TCP port
18
18
  -s--server* Server type: thin, webrick, unicorn, etc
19
19
  -f--finder Start server with finder functionality
20
+ -fs--file_server Activate file serving for resources
20
21
  -R--Rserve_session* Rserve session to use, otherwise start new one
21
22
  -wd--workdir* Change the working directory of the workflow
22
23
  --views* Directory with view templates
@@ -57,9 +58,11 @@ app_dir = app_dir[app]
57
58
  server = options[:server] || 'puma'
58
59
  Misc.in_dir(app_dir) do
59
60
  require 'rack'
60
- ENV["RBBT_FINDER"] = "true" if options.include?(:finder)
61
+ ENV["RBBT_FINDER"] = "true" if options[:finder]
61
62
  ENV["RACK_ENV"] = options[:environment] if options.include?(:environment)
62
63
  ENV["RBBT_VIEWS_DIR"] = options[:views] if options.include?(:views)
64
+ ENV["RBBT_REST_FILE_SERVER"] = "true" if options[:file_server]
65
+
63
66
 
64
67
  if options[:stream]
65
68
  #raise "No streaming available for any server other than puma" unless options[:server] =~ /^puma/
@@ -32,101 +32,6 @@ if options[:help]
32
32
  exit 0
33
33
  end
34
34
 
35
- excludes = %w(.save .crap .source tmp filecache open-remote)
36
- excludes += (options[:exclude] || "").split(/,\s*/)
37
- excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
38
-
39
- test_str = options[:test] ? '-nv' : ''
40
-
41
35
  path, search_path, _sep, *other = ARGV
42
36
 
43
- resource = Rbbt
44
-
45
- paths = if options[:source]
46
- SSHClient.run(options[:source], <<-EOF).split("\n")
47
- require 'rbbt-util'
48
- require 'rbbt/workflow'
49
-
50
- path = "#{path}"
51
- if File.exists?(path)
52
- path = #{resource.to_s}.identify(path)
53
- else
54
- path = Path.setup(path)
55
- end
56
- files = path.glob_all
57
- if #{options[:recursive].to_s == 'true'}
58
- files = Step.job_files_for_archive(files)
59
- end
60
- puts files * "\n"
61
- EOF
62
- else
63
- if File.exists?(path)
64
- path = resource.identify(path)
65
- else
66
- path = Path.setup(path)
67
- end
68
- files = path.glob_all
69
- if options[:recursive]
70
- files = Step.job_files_for_archive(files)
71
- end
72
- files
73
- end
74
-
75
- target = if options[:target]
76
- target = SSHClient.run(options[:target], <<-EOF).split("\n").first
77
- require 'rbbt-util'
78
- path = "var/jobs"
79
- resource = #{resource.to_s}
80
- search_path = "#{search_path}"
81
- puts resource[path].find(search_path)
82
- EOF
83
- else
84
- resource['var/jobs'].find(search_path)
85
- end
86
-
87
- subpath_files = {}
88
- paths.each do |source|
89
- parts = source.split("/")
90
- subpath = parts[0..-4] * "/"
91
- source = parts[-3..-1] * "/"
92
- subpath_files[subpath] ||= []
93
- subpath_files[subpath] << source
94
- end
95
-
96
- subpath_files.each do |subpath, files|
97
- if options[:target]
98
- CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
99
- else
100
- Open.mkdir File.dirname(target)
101
- end
102
-
103
- if options[:source]
104
- source = [options[:source], subpath] * ":"
105
- else
106
- source = subpath
107
- end
108
- target = [options[:target], target] * ":" if options[:target]
109
-
110
- files_and_dirs = Set.new(files )
111
- files.each do |file|
112
- parts = file.split("/")[0..-2]
113
- while parts.any?
114
- files_and_dirs << parts * "/"
115
- parts.pop
116
- end
117
- end
118
-
119
- TmpFile.with_file(files_and_dirs.to_a * "\n") do |tmp_include_file|
120
- includes_str = "--include-from='#{tmp_include_file}'"
121
- cmd = "rsync -atAXmPL --progress #{test_str} --include-from='#{tmp_include_file}' --exclude='*' #{source}/ #{target}/ #{other * " "}"
122
-
123
- cmd << " && rm -Rf #{source}" if options[:delete]
124
-
125
- if options[:print]
126
- ppp Open.read(tmp_include_file)
127
- puts cmd
128
- else
129
- CMD.cmd_log(cmd)
130
- end
131
- end
132
- end
37
+ Step.migrate(path, search_path, options)
@@ -0,0 +1,36 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rbbt-util'
4
+ require 'rbbt/util/simpleopt'
5
+ require 'rbbt/workflow'
6
+ require 'rbbt/workflow/remote/ssh/get'
7
+
8
+ $0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
9
+
10
+ options = SOPT.setup <<EOF
11
+
12
+ Remove a job and its dependencies
13
+
14
+ $ rbbt purge [options] <job_path>
15
+
16
+ You can specify extra rsync options after --
17
+
18
+ -h--help Print this help
19
+ -t--test Do a verbose dry run
20
+ -r--relocate Include relocated dependencies
21
+
22
+ EOF
23
+ if options[:help]
24
+ if defined? rbbt_usage
25
+ rbbt_usage
26
+ else
27
+ puts SOPT.doc
28
+ end
29
+ exit 0
30
+ end
31
+
32
+ path, search_path, _sep, *other = ARGV
33
+
34
+ raise ParameterException, "No path given" if path.nil?
35
+
36
+ Step.purge(path, options[:relocate])
@@ -23,6 +23,7 @@ $ rbbt workflow server [options] <Workflow>
23
23
  -R--requires* Require a list of files
24
24
  --views* Directory with view templates
25
25
  --stream Activate streaming of workflow tasks
26
+ -fs--file_server Activate file serving for resources
26
27
  --export_all Export all workflow tasks (use with caution!)
27
28
  --export* Export workflow tasks (asynchronous)
28
29
  --export_asynchronous* Export workflow tasks as asynchronous
@@ -81,9 +82,10 @@ TmpFile.with_file do |app_dir|
81
82
  Open.write(app_dir.etc.requires.find, requires.split(/,\s*/)*"\n") if requires and not requires.empty?
82
83
 
83
84
  require 'rack'
84
- ENV["RBBT_FINDER"] = "true" if options.include?(:finder)
85
+ ENV["RBBT_FINDER"] = "true" if options[:finder]
85
86
  ENV["RACK_ENV"] = options[:environment] if options.include?(:environment)
86
87
  ENV["RBBT_VIEWS_DIR"] = options[:views] if options.include?(:views)
88
+ ENV["RBBT_REST_FILE_SERVER"] = "true" if options[:file_server]
87
89
 
88
90
  if options[:export_all]
89
91
  ENV["RBBT_WORKFLOW_EXPORT_ALL"] = 'true'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.26.78
4
+ version: 5.26.79
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-09-25 00:00:00.000000000 Z
11
+ date: 2019-10-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -367,6 +367,7 @@ files:
367
367
  - share/rbbt_commands/log
368
368
  - share/rbbt_commands/migrate
369
369
  - share/rbbt_commands/migrate_job
370
+ - share/rbbt_commands/purge_job
370
371
  - share/rbbt_commands/resource/exists
371
372
  - share/rbbt_commands/resource/find
372
373
  - share/rbbt_commands/resource/get