rbbt-util 5.13.32 → 5.13.33
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/rbbt +13 -12
- data/lib/rbbt/monitor.rb +16 -10
- data/lib/rbbt/persist.rb +25 -84
- data/lib/rbbt/persist/tsv/adapter.rb +7 -4
- data/lib/rbbt/tsv/parallel/traverse.rb +17 -1
- data/lib/rbbt/util/concurrency/processes/worker.rb +7 -0
- data/lib/rbbt/util/log.rb +6 -6
- data/lib/rbbt/util/log/progress.rb +1 -1
- data/lib/rbbt/util/log/progress/report.rb +3 -1
- data/lib/rbbt/util/misc/lock.rb +14 -11
- data/lib/rbbt/util/misc/pipes.rb +16 -5
- data/lib/rbbt/util/open.rb +5 -5
- data/lib/rbbt/workflow/step.rb +6 -0
- data/lib/rbbt/workflow/step/run.rb +1 -1
- data/share/rbbt_commands/system/clean +6 -2
- data/share/rbbt_commands/system/status +10 -3
- data/test/rbbt/test_monitor.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ac193605daa68792b87762023e090b4f20b51636
|
4
|
+
data.tar.gz: cff3c91f834c396cfbe8c0c9f65cd046aaff8ffc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 53ca452d6922e58d4f447b37c8dbe42580bc72ee470b2400eeb5cfd1e046f29ec42432295670203040fddf95d546e5f83059aee1e3044f501b8b9d9cab3195be
|
7
|
+
data.tar.gz: 882d73bd8b7b3c025ba441af4f42dfc7b047a7f29ca4aa14456e9733e8ce82e863956c987d35a077d6a6a69bd8e0a9189419fc6f7b96d84bc4cae0d7deb34c4d
|
data/bin/rbbt
CHANGED
@@ -46,18 +46,6 @@ $ rbbt <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-value
|
|
46
46
|
--no_lock_id #{Log.color :yellow, "Do not track lockfiles with ids (for high-througput and high-concurrency"}
|
47
47
|
EOF
|
48
48
|
|
49
|
-
if options.delete(:no_lock_id)
|
50
|
-
Misc.use_lock_id = false
|
51
|
-
end
|
52
|
-
|
53
|
-
if mem_dump = options.delete(:dump_mem)
|
54
|
-
require 'rbbt/monitor'
|
55
|
-
Rbbt.dump_memory(mem_dump, Symbol)
|
56
|
-
end
|
57
|
-
|
58
|
-
if options.delete :nobar
|
59
|
-
ENV["RBBT_NO_PROGRESS"] = "true"
|
60
|
-
end
|
61
49
|
|
62
50
|
locate = options.delete :locate_file
|
63
51
|
|
@@ -73,6 +61,19 @@ else
|
|
73
61
|
end
|
74
62
|
end
|
75
63
|
|
64
|
+
if options.delete(:no_lock_id)
|
65
|
+
Misc.use_lock_id = false
|
66
|
+
end
|
67
|
+
|
68
|
+
if mem_dump = options.delete(:dump_mem)
|
69
|
+
require 'rbbt/monitor'
|
70
|
+
Rbbt.dump_memory(mem_dump, Symbol)
|
71
|
+
end
|
72
|
+
|
73
|
+
if options.delete :nobar
|
74
|
+
ENV["RBBT_NO_PROGRESS"] = "true"
|
75
|
+
end
|
76
|
+
|
76
77
|
if options[:command_dir]
|
77
78
|
$rbbt_command_dir = Path.setup(options[:command_dir].dup)
|
78
79
|
else
|
data/lib/rbbt/monitor.rb
CHANGED
@@ -3,7 +3,7 @@ require 'rbbt'
|
|
3
3
|
module Rbbt
|
4
4
|
|
5
5
|
LOCK_DIRS = Rbbt.share.find_all + Rbbt.var.cache.persistence.find_all + Rbbt.var.jobs.find_all +
|
6
|
-
Rbbt.tmp.tsv_open_locks.find_all + Rbbt.tmp.persist_locks.find_all
|
6
|
+
Rbbt.tmp.tsv_open_locks.find_all + Rbbt.tmp.persist_locks.find_all + Rbbt.tmp.sensiblewrite_lock_dir.find_all
|
7
7
|
|
8
8
|
SENSIBLE_WRITE_DIRS = Misc.sensiblewrite_dir.find_all
|
9
9
|
|
@@ -31,10 +31,15 @@ module Rbbt
|
|
31
31
|
end
|
32
32
|
|
33
33
|
def self.file_time(file)
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
34
|
+
info = {}
|
35
|
+
begin
|
36
|
+
info[:ctime] = File.ctime file
|
37
|
+
info[:atime] = File.atime file
|
38
|
+
info[:elapsed] = Time.now - info[:ctime]
|
39
|
+
rescue Exception
|
40
|
+
end
|
41
|
+
info[:ctime] = Time.now - 999
|
42
|
+
info
|
38
43
|
end
|
39
44
|
|
40
45
|
#{{{ LOCKS
|
@@ -49,17 +54,18 @@ module Rbbt
|
|
49
54
|
def self.lock_info(dirs = LOCK_DIRS)
|
50
55
|
lock_info = {}
|
51
56
|
locks(dirs).each do |f|
|
57
|
+
lock_info[f] = {}
|
52
58
|
begin
|
53
|
-
|
59
|
+
lock_info[f].merge!(file_time(f))
|
54
60
|
if File.size(f) > 0
|
55
61
|
info = Open.open(f) do |s|
|
56
62
|
YAML.load(s)
|
57
63
|
end
|
58
|
-
|
59
|
-
|
64
|
+
IndiferentHash.setup(info)
|
65
|
+
lock_info[f][:pid] = info[:pid]
|
66
|
+
lock_info[f][:ppid] = info[:ppid]
|
60
67
|
end
|
61
|
-
|
62
|
-
rescue
|
68
|
+
rescue Exception
|
63
69
|
Log.exception $!
|
64
70
|
end
|
65
71
|
end
|
data/lib/rbbt/persist.rb
CHANGED
@@ -131,7 +131,7 @@ module Persist
|
|
131
131
|
end
|
132
132
|
end
|
133
133
|
|
134
|
-
def self.save_file(path, type, content)
|
134
|
+
def self.save_file(path, type, content, lockfile = nil)
|
135
135
|
|
136
136
|
return if content.nil?
|
137
137
|
|
@@ -139,17 +139,17 @@ module Persist
|
|
139
139
|
when :nil
|
140
140
|
nil
|
141
141
|
when :boolean
|
142
|
-
Misc.sensiblewrite(path, content ? "true" : "false")
|
142
|
+
Misc.sensiblewrite(path, content ? "true" : "false", :lock => lockfile)
|
143
143
|
when :fwt
|
144
144
|
content.file.seek 0
|
145
|
-
Misc.sensiblewrite(path, content.file.read)
|
145
|
+
Misc.sensiblewrite(path, content.file.read, :lock => lockfile)
|
146
146
|
when :tsv
|
147
147
|
content = content.to_s if TSV === content
|
148
148
|
Misc.sensiblewrite(path, content)
|
149
149
|
when :annotations
|
150
|
-
Misc.sensiblewrite(path, Annotated.tsv(content, :all).to_s)
|
150
|
+
Misc.sensiblewrite(path, Annotated.tsv(content, :all).to_s, :lock => lockfile)
|
151
151
|
when :string, :text
|
152
|
-
Misc.sensiblewrite(path, content)
|
152
|
+
Misc.sensiblewrite(path, content, :lock => lockfile)
|
153
153
|
when :binary
|
154
154
|
content.force_encoding("ASCII-8BIT") if content.respond_to? :force_encoding
|
155
155
|
f = File.open(path, 'wb')
|
@@ -160,38 +160,36 @@ module Persist
|
|
160
160
|
case content
|
161
161
|
when Array
|
162
162
|
if content.empty?
|
163
|
-
Misc.sensiblewrite(path, "")
|
163
|
+
Misc.sensiblewrite(path, "", :lock => lockfile)
|
164
164
|
else
|
165
|
-
Misc.sensiblewrite(path, content * "\n" + "\n")
|
165
|
+
Misc.sensiblewrite(path, content * "\n" + "\n", :lock => lockfile)
|
166
166
|
end
|
167
167
|
when IO
|
168
|
-
Misc.sensiblewrite(path, content)
|
168
|
+
Misc.sensiblewrite(path, content, :lock => lockfile)
|
169
169
|
else
|
170
|
-
Misc.sensiblewrite(path, content.to_s)
|
170
|
+
Misc.sensiblewrite(path, content.to_s, :lock => lockfile)
|
171
171
|
end
|
172
172
|
when :marshal_tsv
|
173
|
-
Misc.sensiblewrite(path, Marshal.dump(content.dup))
|
173
|
+
Misc.sensiblewrite(path, Marshal.dump(content.dup), :lock => lockfile)
|
174
174
|
when :marshal
|
175
175
|
dump = Marshal.dump(content)
|
176
|
-
Misc.sensiblewrite(path, [dump].pack("m"))
|
176
|
+
Misc.sensiblewrite(path, [dump].pack("m"), :lock => lockfile)
|
177
177
|
when :yaml
|
178
|
-
Misc.sensiblewrite(path, YAML.dump(content))
|
178
|
+
Misc.sensiblewrite(path, YAML.dump(content), :lock => lockfile)
|
179
179
|
when :float, :integer, :tsv
|
180
|
-
Misc.sensiblewrite(path, content.to_s)
|
180
|
+
Misc.sensiblewrite(path, content.to_s, :lock => lockfile)
|
181
181
|
else
|
182
182
|
raise "Unknown persistence: #{ type }"
|
183
183
|
end
|
184
184
|
end
|
185
185
|
|
186
|
-
def self.tee_stream_thread(stream, path, type, callback = nil, abort_callback = nil)
|
186
|
+
def self.tee_stream_thread(stream, path, type, callback = nil, abort_callback = nil, lockfile = nil)
|
187
187
|
file, out = Misc.tee_stream(stream)
|
188
188
|
|
189
189
|
saver_thread = Thread.new(Thread.current) do |parent|
|
190
190
|
begin
|
191
191
|
Thread.current["name"] = "file saver: " + path
|
192
|
-
|
193
|
-
save_file(path, type, file)
|
194
|
-
end
|
192
|
+
save_file(path, type, file, lockfile)
|
195
193
|
rescue Aborted
|
196
194
|
Log.medium "Persist stream thread aborted: #{ Log.color :blue, path }"
|
197
195
|
file.abort if file.respond_to? :abort
|
@@ -209,66 +207,6 @@ module Persist
|
|
209
207
|
out
|
210
208
|
end
|
211
209
|
|
212
|
-
def self.tee_stream_pipe(stream, path, type, callback = nil, abort_callback = nil)
|
213
|
-
parent_pid = Process.pid
|
214
|
-
out = Misc.open_pipe true, false do |sin|
|
215
|
-
begin
|
216
|
-
file, out = Misc.tee_stream(stream)
|
217
|
-
|
218
|
-
saver_th = Thread.new(Thread.current, path, file) do |parent,path,file|
|
219
|
-
begin
|
220
|
-
Misc.lock(path) do
|
221
|
-
save_file(path, type, file)
|
222
|
-
end
|
223
|
-
Log.high "Stream pipe saved: #{path}"
|
224
|
-
rescue Aborted
|
225
|
-
Log.medium "Persist stream pipe exception: #{ Log.color :blue, path }"
|
226
|
-
stream.abort if stream.respond_to? :abort
|
227
|
-
rescue Exception
|
228
|
-
Log.medium "Persist stream pipe exception: #{ Log.color :blue, path }"
|
229
|
-
Log.exception $!
|
230
|
-
stream.abort if stream.respond_to? :abort
|
231
|
-
stream.join if stream.respond_to? :join
|
232
|
-
raise $!
|
233
|
-
end
|
234
|
-
end
|
235
|
-
|
236
|
-
tee_th = Thread.new(Thread.current) do |parent|
|
237
|
-
begin
|
238
|
-
while block = out.read(2028)
|
239
|
-
sin.write block
|
240
|
-
end
|
241
|
-
rescue Aborted
|
242
|
-
Log.medium "Tee stream thread aborted"
|
243
|
-
sout.abort if sout.respond_to? :abort
|
244
|
-
sin.abort if sin.respond_to? :abort
|
245
|
-
rescue Exception
|
246
|
-
sin.abort if sin.respond_to? :abort
|
247
|
-
sout.abort if sout.respond_to? :abort
|
248
|
-
Log.exception $!
|
249
|
-
raise $!
|
250
|
-
ensure
|
251
|
-
sin.close unless sin.closed?
|
252
|
-
end
|
253
|
-
end
|
254
|
-
saver_th.join
|
255
|
-
tee_th.join
|
256
|
-
rescue Aborted
|
257
|
-
tee_th.raise Aborted.new if tee_th and tee_th.alive?
|
258
|
-
saver_th.raise Aborted.new if saver_th and saver_th.alive?
|
259
|
-
Kernel.exit! -1
|
260
|
-
rescue Exception
|
261
|
-
tee_th.raise Aborted.new if tee_th and tee_th.alive?
|
262
|
-
saver_th.raise Aborted.new if saver_th and saver_th.alive?
|
263
|
-
Log.exception $!
|
264
|
-
Process.kill :INT, parent_pid
|
265
|
-
Kernel.exit! -1
|
266
|
-
end
|
267
|
-
end
|
268
|
-
stream.close
|
269
|
-
out
|
270
|
-
end
|
271
|
-
|
272
210
|
class << self
|
273
211
|
alias tee_stream tee_stream_thread
|
274
212
|
end
|
@@ -280,7 +218,7 @@ module Persist
|
|
280
218
|
|
281
219
|
if stream
|
282
220
|
if persist_options[:no_load] == :stream
|
283
|
-
res = tee_stream(stream, path, type, stream.respond_to?(:callback)? stream.callback : nil, stream.respond_to?(:abort_callback)? stream.abort_callback : nil)
|
221
|
+
res = tee_stream(stream, path, type, stream.respond_to?(:callback)? stream.callback : nil, stream.respond_to?(:abort_callback)? stream.abort_callback : nil, lockfile)
|
284
222
|
res.lockfile = lockfile
|
285
223
|
|
286
224
|
raise KeepLocked.new res
|
@@ -321,10 +259,11 @@ module Persist
|
|
321
259
|
Open.rm path if Open.exists? path
|
322
260
|
end
|
323
261
|
|
324
|
-
|
325
262
|
lock_filename = Persist.persistence_path(path + '.persist', {:dir => Persist.lock_dir})
|
326
263
|
begin
|
327
|
-
Misc.
|
264
|
+
lock_options = Misc.pull_keys persist_options, :lock
|
265
|
+
lock_options = lock_options[:lock] if Hash === lock_options[:lock]
|
266
|
+
Misc.lock lock_filename, lock_options do |lockfile|
|
328
267
|
|
329
268
|
Misc.insist do
|
330
269
|
if is_persisted?(path, persist_options)
|
@@ -338,9 +277,7 @@ module Persist
|
|
338
277
|
|
339
278
|
res = get_result(path, type, persist_options, lockfile, &block)
|
340
279
|
|
341
|
-
|
342
|
-
save_file(path, type, res)
|
343
|
-
end
|
280
|
+
save_file(path, type, res, lockfile)
|
344
281
|
|
345
282
|
return path if persist_options[:no_load]
|
346
283
|
|
@@ -348,7 +285,11 @@ module Persist
|
|
348
285
|
end
|
349
286
|
|
350
287
|
rescue Lockfile::StolenLockError
|
351
|
-
|
288
|
+
begin
|
289
|
+
Log.medium "Lockfile stolen: #{path}"
|
290
|
+
sleep 1 + rand(2)
|
291
|
+
rescue Exception
|
292
|
+
end
|
352
293
|
retry
|
353
294
|
rescue Exception
|
354
295
|
Log.medium "Error in persist: #{path}#{Open.exists?(path) ? Log.color(:red, " Erasing") : ""}"
|
@@ -54,8 +54,8 @@ module Persist
|
|
54
54
|
|
55
55
|
def write_and_read
|
56
56
|
lock_filename = Persist.persistence_path(persistence_path + '.write', {:dir => TSV.lock_dir})
|
57
|
-
|
58
|
-
|
57
|
+
mutex.synchronize do
|
58
|
+
Misc.lock(lock_filename, true, :max_age => 600) do
|
59
59
|
write if closed? or not write?
|
60
60
|
res = begin
|
61
61
|
yield
|
@@ -69,11 +69,14 @@ module Persist
|
|
69
69
|
|
70
70
|
def write_and_close
|
71
71
|
lock_filename = Persist.persistence_path(persistence_path + '.write', {:dir => TSV.lock_dir})
|
72
|
-
|
73
|
-
|
72
|
+
mutex.synchronize do
|
73
|
+
Misc.lock(lock_filename, true) do
|
74
74
|
write if closed? or not write?
|
75
75
|
res = begin
|
76
76
|
yield
|
77
|
+
rescue Exception
|
78
|
+
Log.exception $!
|
79
|
+
raise $!
|
77
80
|
ensure
|
78
81
|
close
|
79
82
|
end
|
@@ -180,9 +180,14 @@ module TSV
|
|
180
180
|
|
181
181
|
if callback
|
182
182
|
TSV::Parser.traverse(io, options) do |k,v|
|
183
|
-
|
183
|
+
begin
|
184
|
+
callback.call yield k, v
|
185
|
+
ensure
|
186
|
+
bar.tick if bar
|
187
|
+
end
|
184
188
|
end
|
185
189
|
else
|
190
|
+
options[:monitor] = bar
|
186
191
|
TSV::Parser.traverse(io, options, &block)
|
187
192
|
end
|
188
193
|
join.call if join
|
@@ -492,6 +497,17 @@ module TSV
|
|
492
497
|
threads = nil if threads and threads.to_i <= 1
|
493
498
|
cpus = nil if cpus and cpus.to_i <= 1
|
494
499
|
|
500
|
+
if options[:keys]
|
501
|
+
case options[:keys]
|
502
|
+
when TrueClass
|
503
|
+
options[:type] = :keys
|
504
|
+
when String
|
505
|
+
options[:type] = :keys
|
506
|
+
options[:key_field] = options[:keys]
|
507
|
+
options[:fields] = []
|
508
|
+
end
|
509
|
+
end
|
510
|
+
|
495
511
|
bar = Misc.process_options options, :bar
|
496
512
|
bar ||= Misc.process_options options, :progress
|
497
513
|
max = guess_max(obj)
|
@@ -7,6 +7,13 @@ class RbbtProcessQueue
|
|
7
7
|
|
8
8
|
@pid = Process.fork do
|
9
9
|
begin
|
10
|
+
Persist::CONNECTIONS.values.each do |db| db.close if db.write? end
|
11
|
+
ObjectSpace.each_object(Mutex) do |m|
|
12
|
+
begin
|
13
|
+
m.unlock
|
14
|
+
rescue ThreadError
|
15
|
+
end if m.locked?
|
16
|
+
end
|
10
17
|
@cleanup.call if @cleanup
|
11
18
|
@queue.close_write
|
12
19
|
|
data/lib/rbbt/util/log.rb
CHANGED
@@ -133,12 +133,12 @@ module Log
|
|
133
133
|
message = "" << highlight << message << color(0) if severity >= INFO
|
134
134
|
str = prefix << " " << message
|
135
135
|
|
136
|
-
LOG_MUTEX.synchronize do
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
end
|
136
|
+
#LOG_MUTEX.synchronize do
|
137
|
+
STDERR.puts str
|
138
|
+
Log::LAST.replace "log"
|
139
|
+
logfile.puts str unless logfile.nil?
|
140
|
+
nil
|
141
|
+
#end
|
142
142
|
end
|
143
143
|
|
144
144
|
def self.log_obj_inspect(obj, level, file = $stdout)
|
@@ -89,12 +89,14 @@ module Log
|
|
89
89
|
def report(io = STDERR)
|
90
90
|
if Log::LAST != "progress"
|
91
91
|
length = Log::ProgressBar.cleanup_bars
|
92
|
-
print(io, Log.color(:yellow, "--Progress\n"))
|
93
92
|
bars = BARS
|
94
93
|
bars.sort_by{|b| b.depth }.reverse.each do |bar|
|
95
94
|
print(io, Log.color(:yellow ,bar.report_msg) << "\n")
|
96
95
|
end
|
96
|
+
else
|
97
|
+
bars = BARS
|
97
98
|
end
|
99
|
+
print(io, up_lines(bars.length) << Log.color(:yellow, "--Progress\n") << down_lines(bars.length))
|
98
100
|
print(io, up_lines(@depth) << report_msg << down_lines(@depth))
|
99
101
|
@last_time = Time.now
|
100
102
|
@last_count = ticks
|
data/lib/rbbt/util/misc/lock.rb
CHANGED
@@ -8,12 +8,15 @@ module Misc
|
|
8
8
|
Lockfile.suspend = 10
|
9
9
|
else
|
10
10
|
Log.medium "De-activating lockfile ids"
|
11
|
-
Lockfile.dont_use_lock_id = false
|
12
11
|
Lockfile.dont_use_lock_id = true
|
13
|
-
Lockfile.refresh =
|
14
|
-
Lockfile.max_age =
|
15
|
-
Lockfile.suspend =
|
12
|
+
Lockfile.refresh = 5
|
13
|
+
Lockfile.max_age = 30
|
14
|
+
Lockfile.suspend = 5
|
16
15
|
end
|
16
|
+
|
17
|
+
Lockfile.refresh = 3
|
18
|
+
Lockfile.max_age = 10
|
19
|
+
Lockfile.suspend = 2
|
17
20
|
end
|
18
21
|
|
19
22
|
self.use_lock_id = ENV["RBBT_NO_LOCKFILE_ID"] != "true"
|
@@ -26,19 +29,19 @@ module Misc
|
|
26
29
|
res = nil
|
27
30
|
|
28
31
|
lock_path = File.expand_path(file + '.lock')
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
+
if options[:lock]
|
33
|
+
lockfile = options[:lock]
|
34
|
+
lockfile.lock unless lockfile.locked?
|
35
|
+
else
|
36
|
+
lockfile = Lockfile.new(lock_path, options)
|
37
|
+
lockfile.lock
|
38
|
+
end
|
32
39
|
|
33
40
|
begin
|
34
41
|
res = yield lockfile
|
35
|
-
#rescue Lockfile::StolenLockError
|
36
42
|
rescue KeepLocked
|
37
43
|
unlock = false
|
38
44
|
res = $!.payload
|
39
|
-
rescue Exception
|
40
|
-
lockfile.unlock #if lockfile.locked?
|
41
|
-
raise $!
|
42
45
|
ensure
|
43
46
|
if unlock
|
44
47
|
begin
|
data/lib/rbbt/util/misc/pipes.rb
CHANGED
@@ -1,4 +1,13 @@
|
|
1
|
+
require 'rbbt'
|
2
|
+
|
1
3
|
module Misc
|
4
|
+
class << self
|
5
|
+
attr_accessor :sensiblewrite_lock_dir
|
6
|
+
|
7
|
+
def sensible_write_locks
|
8
|
+
@sensiblewrite_locks ||= Rbbt.tmp.sensiblewrite_locks.find
|
9
|
+
end
|
10
|
+
end
|
2
11
|
|
3
12
|
class << self
|
4
13
|
attr_accessor :sensiblewrite_dir
|
@@ -112,7 +121,7 @@ module Misc
|
|
112
121
|
stream.abort if stream.respond_to? :abort
|
113
122
|
stream_out1.abort if stream_out1.respond_to? :abort
|
114
123
|
stream_out2.abort if stream_out2.respond_to? :abort
|
115
|
-
stream.join
|
124
|
+
stream.join if stream.respond_to? :join
|
116
125
|
Log.medium "Exception in tee_stream_thread: #{$!.message}"
|
117
126
|
raise $!
|
118
127
|
end
|
@@ -199,10 +208,12 @@ module Misc
|
|
199
208
|
str
|
200
209
|
end
|
201
210
|
|
202
|
-
def self.sensiblewrite(path, content = nil, &block)
|
211
|
+
def self.sensiblewrite(path, content = nil, options = {}, &block)
|
212
|
+
lock_options = Misc.pull_keys options, :lock
|
213
|
+
lock_options = lock_options[:lock] if Hash === lock_options[:lock]
|
203
214
|
return if Open.exists? path
|
204
|
-
tmp_path = Persist.persistence_path(path, {:dir => Misc.
|
205
|
-
Misc.lock tmp_path do
|
215
|
+
tmp_path = Persist.persistence_path(path, {:dir => Misc.sensiblewrite_lock_dir})
|
216
|
+
Misc.lock tmp_path, lock_options do
|
206
217
|
return if Open.exists? path
|
207
218
|
if not Open.exists? path
|
208
219
|
FileUtils.rm_f tmp_path if File.exists? tmp_path
|
@@ -231,7 +242,7 @@ module Misc
|
|
231
242
|
File.open(tmp_path, 'wb') do |f| end
|
232
243
|
end
|
233
244
|
|
234
|
-
Open.mv tmp_path, path
|
245
|
+
Open.mv tmp_path, path, lock_options
|
235
246
|
content.join if content.respond_to? :join
|
236
247
|
rescue Aborted
|
237
248
|
Log.medium "Aborted sensiblewrite -- #{ Log.reset << Log.color(:blue, path) }"
|
data/lib/rbbt/util/open.rb
CHANGED
@@ -153,14 +153,14 @@ module Open
|
|
153
153
|
|
154
154
|
def self.save_content_in_repo(dir, sub_path, content)
|
155
155
|
repo = get_repo_from_dir(dir)
|
156
|
-
repo.
|
156
|
+
repo.write_and_close do
|
157
157
|
repo[sub_path] = content
|
158
158
|
end
|
159
159
|
end
|
160
160
|
|
161
161
|
def self.remove_from_repo(dir, sub_path, recursive = false)
|
162
162
|
repo = get_repo_from_dir(dir)
|
163
|
-
repo.
|
163
|
+
repo.write_and_close do
|
164
164
|
if recursive
|
165
165
|
repo.outlist repo.range sub_path, true, sub_path.sub(/.$/,('\1'.ord + 1).chr), false
|
166
166
|
else
|
@@ -233,7 +233,7 @@ module Open
|
|
233
233
|
end
|
234
234
|
end
|
235
235
|
|
236
|
-
def self.mv(source, target)
|
236
|
+
def self.mv(source, target, options)
|
237
237
|
dir_sub_path_source = find_repo_dir(source)
|
238
238
|
dir_sub_path_target = find_repo_dir(target)
|
239
239
|
|
@@ -252,8 +252,8 @@ module Open
|
|
252
252
|
repo_source = get_repo_from_dir(dir_sub_path_source[0])
|
253
253
|
repo_target = get_repo_from_dir(dir_sub_path_target[0])
|
254
254
|
|
255
|
-
repo_source.
|
256
|
-
repo_target.
|
255
|
+
repo_source.write_and_close do
|
256
|
+
repo_target.write_and_close do
|
257
257
|
repo_source[dir_sub_path_source[1]] = repo_target[dir_sub_path_target[1]]
|
258
258
|
end
|
259
259
|
end
|
data/lib/rbbt/workflow/step.rb
CHANGED
@@ -35,6 +35,12 @@ class Step
|
|
35
35
|
@mutex = Mutex.new
|
36
36
|
@info_mutex = Mutex.new
|
37
37
|
@inputs = inputs || []
|
38
|
+
NamedArray.setup @inputs, task.inputs
|
39
|
+
end
|
40
|
+
|
41
|
+
def inputs
|
42
|
+
NamedArray.setup @inputs, task.inputs if task.inputs and not NamedArray === @inputs
|
43
|
+
@inputs
|
38
44
|
end
|
39
45
|
|
40
46
|
def task_name
|
@@ -92,7 +92,7 @@ class Step
|
|
92
92
|
@result = self._exec
|
93
93
|
@result = @result.stream if TSV::Dumper === @result
|
94
94
|
end
|
95
|
-
no_load ? @result : prepare_result(@result, @task.result_description)
|
95
|
+
no_load or ENV["RBBT_NO_STREAM"] ? @result : prepare_result(@result, @task.result_description)
|
96
96
|
end
|
97
97
|
|
98
98
|
def checks
|
@@ -34,7 +34,7 @@ if locks.any?
|
|
34
34
|
puts
|
35
35
|
puts Log.color(:magenta, "Locks:")
|
36
36
|
locks.each do |file,info|
|
37
|
-
if force or (info[:pid] and Misc.pid_exists? info[:pid])
|
37
|
+
if force or (info[:pid] and not Misc.pid_exists? info[:pid])
|
38
38
|
puts " Removing #{ file }"
|
39
39
|
File.unlink file
|
40
40
|
end
|
@@ -65,6 +65,9 @@ if sensiblewrites.any?
|
|
65
65
|
end
|
66
66
|
end
|
67
67
|
|
68
|
+
exit 0 if workflow.nil?
|
69
|
+
workflow = nil if workflow == ["all"]
|
70
|
+
|
68
71
|
puts
|
69
72
|
puts Log.color(:magenta, "# Workflow clean")
|
70
73
|
puts
|
@@ -89,7 +92,8 @@ jobs.each do |file,i|
|
|
89
92
|
status = :dead if status != "done" and pid and not Misc.pid_exists?(pid)
|
90
93
|
|
91
94
|
status = status.to_s
|
95
|
+
next unless status =~ /\berror$/ or status =~ /\bmissing$/ or status =~ /\baborted$/ or status =~ /\bdead$/ or status == "" or (force and status == 'noinfo')
|
92
96
|
puts " Removing #{ file } - #{status}"
|
93
|
-
Step.clean(file)
|
97
|
+
Step.clean(file)
|
94
98
|
end
|
95
99
|
|
@@ -20,6 +20,7 @@ rbbt_usage and exit 0 if options[:help]
|
|
20
20
|
|
21
21
|
workflow, task = ARGV
|
22
22
|
|
23
|
+
|
23
24
|
workflow = workflow.split "," if workflow
|
24
25
|
task = task.split "," if task
|
25
26
|
|
@@ -58,11 +59,16 @@ end
|
|
58
59
|
|
59
60
|
puts Log.color(:magenta, "# System report")
|
60
61
|
puts
|
61
|
-
|
62
|
+
sort_file = Proc.new do |f,i|
|
63
|
+
dir = f.split(/:\_/).first
|
64
|
+
|
65
|
+
dir + i[:ctime].to_i.to_s
|
66
|
+
end
|
62
67
|
locks = Rbbt.lock_info
|
63
68
|
if locks.any?
|
64
69
|
puts Log.color(:magenta, "Locks:")
|
65
|
-
locks.sort_by{|f,i| i[:ctime] }.each do |file,info|
|
70
|
+
#locks.sort_by{|f,i| i[:ctime] }.each do |file,info|
|
71
|
+
locks.sort_by(&sort_file).each do |file,info|
|
66
72
|
elapsed, pid, ppid = info.values_at :elapsed, :pid, :ppid
|
67
73
|
puts " " << file + Log.color(:blue, " -- time: #{Misc.format_seconds elapsed}; ppid: #{ppid}; pid: #{pid_msg pid}")
|
68
74
|
end
|
@@ -90,6 +96,8 @@ if sensiblewrites.any?
|
|
90
96
|
puts
|
91
97
|
end
|
92
98
|
|
99
|
+
exit 0 if workflow.nil?
|
100
|
+
workflow = nil if workflow == ["all"]
|
93
101
|
|
94
102
|
puts Log.color(:magenta, "# Workflows")
|
95
103
|
|
@@ -115,7 +123,6 @@ workflows.sort.each do |workflow,tasks|
|
|
115
123
|
Marshal.load(f)
|
116
124
|
end
|
117
125
|
rescue
|
118
|
-
Log.exception $!
|
119
126
|
{:status => :noinfo}
|
120
127
|
end
|
121
128
|
|
data/test/rbbt/test_monitor.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.13.
|
4
|
+
version: 5.13.33
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-06-
|
11
|
+
date: 2014-06-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|