rbbt-util 5.14.0 → 5.14.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/rbbt +1 -1
- data/lib/rbbt/annotations/annotated_array.rb +15 -4
- data/lib/rbbt/persist.rb +3 -5
- data/lib/rbbt/tsv/dumper.rb +8 -6
- data/lib/rbbt/tsv/parser.rb +1 -0
- data/lib/rbbt/tsv/stream.rb +11 -0
- data/lib/rbbt/tsv/util.rb +2 -0
- data/lib/rbbt/util/log/progress/report.rb +5 -1
- data/lib/rbbt/util/misc/lock.rb +6 -1
- data/lib/rbbt/util/misc/pipes.rb +7 -3
- data/lib/rbbt/util/open.rb +1 -1
- data/lib/rbbt/workflow/accessor.rb +2 -1
- data/lib/rbbt/workflow/step/run.rb +8 -6
- data/test/rbbt/tsv/test_stream.rb +21 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fa73fffbfca850f0304a250527157c35aef1dcb5
|
4
|
+
data.tar.gz: aab332c2a7b11e9d7a207f1640af194dc54a9eb1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 231c27fb1f1b1f81569c0613c4e3b5b61d5d0873bb085671421962934802be074c4a94155bd251d59a37d3098ce2a3729febc759c0320eefa453f9c731faf907
|
7
|
+
data.tar.gz: 394a55fe8949acea897c028106ac47bff2f3f3828a9fab1b0fc44446c19ce0c6d40ba83d5da4c9bba8709e35d425741984ff04d643c3dd270cb9a4fb89a5d9e6
|
data/bin/rbbt
CHANGED
@@ -37,22 +37,33 @@ module AnnotatedArray
|
|
37
37
|
pos = 0
|
38
38
|
super do |value|
|
39
39
|
|
40
|
-
|
40
|
+
case value
|
41
|
+
when Array
|
42
|
+
value = value.dup if value.frozen?
|
43
|
+
|
44
|
+
value = annotate(value)
|
45
|
+
|
46
|
+
value.extend AnnotatedArray if Array === value and Annotated === value
|
47
|
+
|
48
|
+
value.container = self
|
49
|
+
value.container_index = pos
|
50
|
+
|
51
|
+
pos += 1
|
41
52
|
|
42
53
|
block.call value
|
43
|
-
|
54
|
+
when String
|
44
55
|
|
45
56
|
value = value.dup if value.frozen?
|
46
57
|
|
47
58
|
value = annotate(value)
|
48
59
|
|
49
|
-
value.extend AnnotatedArray if Array === value and Annotated === value
|
50
|
-
|
51
60
|
value.container = self
|
52
61
|
value.container_index = pos
|
53
62
|
|
54
63
|
pos += 1
|
55
64
|
|
65
|
+
block.call value
|
66
|
+
else
|
56
67
|
block.call value
|
57
68
|
end
|
58
69
|
end
|
data/lib/rbbt/persist.rb
CHANGED
@@ -132,7 +132,6 @@ module Persist
|
|
132
132
|
end
|
133
133
|
|
134
134
|
def self.save_file(path, type, content, lockfile = nil)
|
135
|
-
|
136
135
|
return if content.nil?
|
137
136
|
|
138
137
|
case (type || :marshal).to_sym
|
@@ -145,7 +144,7 @@ module Persist
|
|
145
144
|
Misc.sensiblewrite(path, content.file.read, :lock => lockfile)
|
146
145
|
when :tsv
|
147
146
|
content = content.to_s if TSV === content
|
148
|
-
Misc.sensiblewrite(path, content)
|
147
|
+
Misc.sensiblewrite(path, content, :lock => lockfile)
|
149
148
|
when :annotations
|
150
149
|
Misc.sensiblewrite(path, Annotated.tsv(content, :all).to_s, :lock => lockfile)
|
151
150
|
when :string, :text
|
@@ -219,8 +218,8 @@ module Persist
|
|
219
218
|
|
220
219
|
if stream
|
221
220
|
if persist_options[:no_load] == :stream
|
222
|
-
res = tee_stream(stream, path, type, stream.respond_to?(:callback)? stream.callback : nil, stream.respond_to?(:abort_callback)? stream.abort_callback : nil)
|
223
|
-
res.lockfile = lockfile
|
221
|
+
res = tee_stream(stream, path, type, stream.respond_to?(:callback)? stream.callback : nil, stream.respond_to?(:abort_callback)? stream.abort_callback : nil, lockfile)
|
222
|
+
#res.lockfile = lockfile
|
224
223
|
|
225
224
|
raise KeepLocked.new res
|
226
225
|
else
|
@@ -265,7 +264,6 @@ module Persist
|
|
265
264
|
lock_options = Misc.pull_keys persist_options, :lock
|
266
265
|
lock_options = lock_options[:lock] if Hash === lock_options[:lock]
|
267
266
|
Misc.lock lock_filename, lock_options do |lockfile|
|
268
|
-
|
269
267
|
Misc.insist do
|
270
268
|
if is_persisted?(path, persist_options)
|
271
269
|
Log.low "Persist up-to-date (suddenly): #{ path } - #{Misc.fingerprint persist_options}"
|
data/lib/rbbt/tsv/dumper.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
module TSV
|
2
2
|
class Dumper
|
3
|
-
attr_accessor :in_stream, :stream, :options, :filename
|
3
|
+
attr_accessor :in_stream, :stream, :options, :filename, :sep
|
4
4
|
def self.stream(options = {}, filename = nil, &block)
|
5
5
|
dumper = TSV::Dumper.new options, filename
|
6
6
|
Thread.new(Thread.current) do |parent|
|
@@ -22,18 +22,19 @@ module TSV
|
|
22
22
|
@stream, @in_stream = Misc.pipe
|
23
23
|
end
|
24
24
|
|
25
|
-
def self.values_to_s(values, fields = nil)
|
25
|
+
def self.values_to_s(values, fields = nil, sep = "\t")
|
26
|
+
sep = "\t" if sep.nil?
|
26
27
|
case values
|
27
28
|
when nil
|
28
29
|
if fields.nil? or fields.empty?
|
29
30
|
"\n"
|
30
31
|
else
|
31
|
-
|
32
|
+
sep + ([""] * fields.length) * sep << "\n"
|
32
33
|
end
|
33
34
|
when Array
|
34
|
-
|
35
|
+
sep + values.collect{|v| Array === v ? v * "|" : v} * sep << "\n"
|
35
36
|
else
|
36
|
-
|
37
|
+
sep + values.to_s << "\n"
|
37
38
|
end
|
38
39
|
end
|
39
40
|
|
@@ -49,9 +50,10 @@ module TSV
|
|
49
50
|
|
50
51
|
def add(k,v)
|
51
52
|
@fields ||= @options[:fields]
|
53
|
+
@sep ||= @options[:sep]
|
52
54
|
begin
|
53
55
|
Thread.pass while IO.select(nil, [@in_stream],nil,1).nil?
|
54
|
-
@in_stream << k << TSV::Dumper.values_to_s(v, @fields)
|
56
|
+
@in_stream << k << TSV::Dumper.values_to_s(v, @fields, @sep)
|
55
57
|
rescue IOError
|
56
58
|
rescue Exception
|
57
59
|
raise $!
|
data/lib/rbbt/tsv/parser.rb
CHANGED
data/lib/rbbt/tsv/stream.rb
CHANGED
@@ -175,4 +175,15 @@ module TSV
|
|
175
175
|
|
176
176
|
out
|
177
177
|
end
|
178
|
+
|
179
|
+
def self.stream_flat2double(stream, options = {})
|
180
|
+
parser = TSV::Parser.new stream
|
181
|
+
dumper_options = parser.options.merge(options).merge(:type => :double)
|
182
|
+
dumper = TSV::Dumper.new dumper_options
|
183
|
+
dumper.init
|
184
|
+
TSV.traverse parser, :into => dumper do |key,values|
|
185
|
+
[key, [values]]
|
186
|
+
end
|
187
|
+
dumper.stream
|
188
|
+
end
|
178
189
|
end
|
data/lib/rbbt/tsv/util.rb
CHANGED
@@ -82,7 +82,11 @@ module Log
|
|
82
82
|
str = Log.color :magenta, desc
|
83
83
|
return str << " " << Log.color(:yellow, "waiting") if @ticks == 0
|
84
84
|
str << " " << thr_msg
|
85
|
-
|
85
|
+
if max
|
86
|
+
str << Log.color(:blue, " -- ") << eta_msg
|
87
|
+
else
|
88
|
+
str << Log.color(:blue, " -- ") << ticks.to_s
|
89
|
+
end
|
86
90
|
str
|
87
91
|
end
|
88
92
|
|
data/lib/rbbt/util/misc/lock.rb
CHANGED
@@ -22,15 +22,20 @@ module Misc
|
|
22
22
|
self.use_lock_id = ENV["RBBT_NO_LOCKFILE_ID"] != "true"
|
23
23
|
|
24
24
|
LOCK_MUTEX = Mutex.new
|
25
|
+
#def self.lock(file, unlock = true, options = {})
|
25
26
|
def self.lock(file, unlock = true, options = {})
|
27
|
+
unlock, options = true, unlock if Hash === unlock
|
26
28
|
return yield if file.nil?
|
27
29
|
FileUtils.mkdir_p File.dirname(File.expand_path(file)) unless File.exists? File.dirname(File.expand_path(file))
|
28
30
|
|
29
31
|
res = nil
|
30
32
|
|
31
|
-
|
33
|
+
case options[:lock]
|
34
|
+
when Lockfile
|
32
35
|
lockfile = options[:lock]
|
33
36
|
lockfile.lock unless lockfile.locked?
|
37
|
+
when FalseClass
|
38
|
+
unlock = false
|
34
39
|
else
|
35
40
|
lock_path = File.expand_path(file + '.lock')
|
36
41
|
lockfile = Lockfile.new(lock_path, options)
|
data/lib/rbbt/util/misc/pipes.rb
CHANGED
@@ -214,9 +214,9 @@ module Misc
|
|
214
214
|
|
215
215
|
def self.sensiblewrite(path, content = nil, options = {}, &block)
|
216
216
|
force = Misc.process_options options, :force
|
217
|
-
lock_options = Misc.pull_keys options, :lock
|
218
|
-
lock_options = lock_options[:lock] if Hash === lock_options[:lock]
|
219
217
|
return if Open.exists? path and not force
|
218
|
+
lock_options = Misc.pull_keys options.dup, :lock
|
219
|
+
lock_options = lock_options[:lock] if Hash === lock_options[:lock]
|
220
220
|
tmp_path = Persist.persistence_path(path, {:dir => Misc.sensiblewrite_dir})
|
221
221
|
tmp_path_lock = Persist.persistence_path(path, {:dir => Misc.sensiblewrite_lock_dir})
|
222
222
|
Misc.lock tmp_path_lock, lock_options do
|
@@ -241,7 +241,11 @@ module Misc
|
|
241
241
|
File.open(tmp_path, 'wb') do |f| end
|
242
242
|
end
|
243
243
|
|
244
|
-
|
244
|
+
begin
|
245
|
+
Open.mv tmp_path, path, lock_options
|
246
|
+
rescue
|
247
|
+
raise $! unless File.exists? path
|
248
|
+
end
|
245
249
|
content.join if content.respond_to? :join
|
246
250
|
rescue Aborted
|
247
251
|
Log.medium "Aborted sensiblewrite -- #{ Log.reset << Log.color(:blue, path) }"
|
data/lib/rbbt/util/open.rb
CHANGED
@@ -11,6 +11,7 @@ class Step
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def self.wait_for_jobs(jobs)
|
14
|
+
jobs = [jobs] if Step === jobs
|
14
15
|
begin
|
15
16
|
threads = []
|
16
17
|
jobs.each do |j| threads << Thread.new{j.join} end
|
@@ -112,7 +113,7 @@ class Step
|
|
112
113
|
i = info(false)
|
113
114
|
i[key] = value
|
114
115
|
@info_cache = i
|
115
|
-
Misc.sensiblewrite(info_file, INFO_SERIALIAZER.dump(i), :force => true)
|
116
|
+
Misc.sensiblewrite(info_file, INFO_SERIALIAZER.dump(i), :force => true, :lock => false)
|
116
117
|
@info_cache_time = Time.now
|
117
118
|
value
|
118
119
|
end
|
@@ -15,9 +15,9 @@ class Step
|
|
15
15
|
Log.medium "Not duplicating stream #{ Misc.fingerprint(stream) }"
|
16
16
|
STREAM_CACHE[stream] = stream
|
17
17
|
when File
|
18
|
-
if Open.exists? current.
|
18
|
+
if Open.exists? current.path
|
19
19
|
Log.medium "Reopening file #{ Misc.fingerprint(current) }"
|
20
|
-
Open.open(current.
|
20
|
+
Open.open(current.path)
|
21
21
|
else
|
22
22
|
Log.medium "Duplicating file #{ Misc.fingerprint(current) } #{current.inspect}"
|
23
23
|
Misc.dup_stream(current)
|
@@ -131,7 +131,7 @@ class Step
|
|
131
131
|
@seen.each do |dependency|
|
132
132
|
next if dependency == self
|
133
133
|
next unless dependencies.include? dependency
|
134
|
-
dependency.relay_log self
|
134
|
+
#dependency.relay_log self
|
135
135
|
dependency.dup_inputs
|
136
136
|
end
|
137
137
|
|
@@ -235,9 +235,11 @@ class Step
|
|
235
235
|
ConcurrentStream.setup stream do
|
236
236
|
begin
|
237
237
|
if status != :done
|
238
|
-
|
239
|
-
|
240
|
-
|
238
|
+
Misc.insist do
|
239
|
+
set_info :done, (done_time = Time.now)
|
240
|
+
set_info :time_elapsed, (time_elapsed = done_time - start_time)
|
241
|
+
log :done, "#{Log.color :magenta, "Completed"} #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i} sec."
|
242
|
+
end
|
241
243
|
end
|
242
244
|
rescue
|
243
245
|
Log.exception $!
|
@@ -175,4 +175,25 @@ row1 A B C
|
|
175
175
|
assert_equal ["A", "B", "C", ""], tsv["row1"]
|
176
176
|
assert_equal ["AA", "BB", "CC", ""], tsv["row2"]
|
177
177
|
end
|
178
|
+
|
179
|
+
def test_flat2double
|
180
|
+
text1=<<-EOF
|
181
|
+
#: :sep= #:type=:flat
|
182
|
+
#Row LabelA
|
183
|
+
row1 A AA AAA
|
184
|
+
row2 a aa aaa
|
185
|
+
EOF
|
186
|
+
|
187
|
+
text2=<<-EOF
|
188
|
+
#: :sep= #:type=:double
|
189
|
+
#Row LabelA
|
190
|
+
row1 A|AA|AAA
|
191
|
+
row2 a|aa|aaa
|
192
|
+
EOF
|
193
|
+
s1 = StringIO.new text1
|
194
|
+
s2 = TSV.stream_flat2double(s1, :sep => " ")
|
195
|
+
|
196
|
+
assert_equal text2, s2.read
|
197
|
+
end
|
198
|
+
|
178
199
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.14.
|
4
|
+
version: 5.14.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-06-
|
11
|
+
date: 2014-06-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|