rbbt-util 5.27.1 → 5.27.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/annotations/util.rb +1 -2
- data/lib/rbbt/entity.rb +8 -5
- data/lib/rbbt/fix_width_table.rb +6 -5
- data/lib/rbbt/knowledge_base/entity.rb +3 -2
- data/lib/rbbt/knowledge_base/query.rb +9 -0
- data/lib/rbbt/persist.rb +2 -3
- data/lib/rbbt/persist/tsv.rb +5 -5
- data/lib/rbbt/persist/tsv/adapter.rb +136 -43
- data/lib/rbbt/persist/tsv/tokyocabinet.rb +5 -3
- data/lib/rbbt/resource.rb +8 -5
- data/lib/rbbt/resource/path.rb +3 -3
- data/lib/rbbt/tsv/accessor.rb +6 -6
- data/lib/rbbt/tsv/change_id.rb +3 -1
- data/lib/rbbt/tsv/parallel/traverse.rb +1 -1
- data/lib/rbbt/tsv/util.rb +1 -0
- data/lib/rbbt/util/misc/exceptions.rb +8 -0
- data/lib/rbbt/workflow.rb +1 -1
- data/lib/rbbt/workflow/accessor.rb +3 -1
- data/lib/rbbt/workflow/definition.rb +6 -0
- data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +9 -3
- data/lib/rbbt/workflow/remote_workflow/remote_step.rb +9 -3
- data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +7 -1
- data/lib/rbbt/workflow/step/accessor.rb +4 -0
- data/lib/rbbt/workflow/step/dependencies.rb +10 -6
- data/lib/rbbt/workflow/step/run.rb +2 -2
- data/lib/rbbt/workflow/task.rb +1 -1
- data/lib/rbbt/workflow/usage.rb +25 -8
- data/lib/rbbt/workflow/util/archive.rb +30 -4
- data/share/install/software/lib/install_helpers +9 -3
- data/share/rbbt_commands/migrate_job +3 -1
- data/share/rbbt_commands/system/status +3 -2
- data/share/rbbt_commands/tsv/get +33 -7
- data/share/rbbt_commands/workflow/monitor +3 -3
- data/share/rbbt_commands/workflow/task +1 -1
- data/test/rbbt/knowledge_base/test_query.rb +1 -1
- data/test/rbbt/test_entity.rb +14 -5
- data/test/rbbt/test_knowledge_base.rb +3 -3
- data/test/rbbt/workflow/step/test_dependencies.rb +40 -8
- data/test/rbbt/workflow/test_remote_workflow.rb +13 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fe7c3dec4d2333e77ea396a2d6e572512bf7033999ff4715f1a1c6c9d3d1b9d5
|
4
|
+
data.tar.gz: 32a3d905c1a95f9c7e5cb359855f8781509cfeecdd8db7b77ee29eaea528d030
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d009024642898f775d423e1ddede6aea538e8ed5ff4675eedadd64768e67b0a46f6519038378a221db76b0e196cfc713ca8dc7553788cdb9a51014c14a97de4c
|
7
|
+
data.tar.gz: 5c0612feca84b00e6d402f7a68ca1cebd6f4b7a785b40899bdaed3385a3f68853824dcc98d9cfd2e641768ef5ca3d52e38e41bdbc1d7a3751ae288e8f8d7ed2d
|
@@ -145,10 +145,9 @@ module Annotated
|
|
145
145
|
|
146
146
|
end
|
147
147
|
|
148
|
-
fields = fields.collect{|f| f.to_s}
|
148
|
+
fields = fields.collect{|f| f.to_s}.uniq
|
149
149
|
|
150
150
|
case
|
151
|
-
|
152
151
|
when (Annotated === annotations and not (AnnotatedArray === annotations and annotations.double_array))
|
153
152
|
tsv = TSV.setup({}, :key_field => "List", :fields => fields, :type => :list, :unnamed => true)
|
154
153
|
|
data/lib/rbbt/entity.rb
CHANGED
@@ -194,13 +194,16 @@ module Entity
|
|
194
194
|
multi_name = "_multiple_" << name
|
195
195
|
|
196
196
|
define_method multi_name do
|
197
|
-
if self.instance_variable_get("@
|
198
|
-
return self.instance_variable_get("@
|
197
|
+
if self.instance_variable_get("@multiple_result_" + name.to_s)
|
198
|
+
return self.instance_variable_get("@multiple_result_" + name.to_s)
|
199
199
|
end
|
200
|
-
raise MultipleEntity, "Entity #{name} runs with multiple entities
|
200
|
+
raise MultipleEntity, "Entity #{name} runs with multiple entities"
|
201
201
|
end
|
202
202
|
|
203
203
|
define_method name do |*args|
|
204
|
+
if self.instance_variable_get("@multiple_result_" + name.to_s)
|
205
|
+
return self.instance_variable_get("@multiple_result_" + name.to_s)
|
206
|
+
end
|
204
207
|
obj = if Array === self
|
205
208
|
self
|
206
209
|
elsif self.respond_to?(:container) && Array === self.container
|
@@ -222,11 +225,11 @@ module Entity
|
|
222
225
|
case res
|
223
226
|
when Array
|
224
227
|
missing.zip(res).each do |o,res|
|
225
|
-
o.instance_variable_set("@
|
228
|
+
o.instance_variable_set("@multiple_result_" + name.to_s, res)
|
226
229
|
end
|
227
230
|
when Hash
|
228
231
|
res.each do |o,res|
|
229
|
-
o.instance_variable_set("@
|
232
|
+
o.instance_variable_set("@multiple_result_" + name.to_s, res)
|
230
233
|
end
|
231
234
|
end
|
232
235
|
|
data/lib/rbbt/fix_width_table.rb
CHANGED
@@ -4,20 +4,21 @@ class FixWidthTable
|
|
4
4
|
def initialize(filename, value_size = nil, range = nil, update = false, in_memory = true)
|
5
5
|
@filename = filename
|
6
6
|
|
7
|
-
if update
|
7
|
+
if update || %w(memory stringio).include?(filename.to_s.downcase) || ! File.exist?(filename)
|
8
8
|
Log.debug "FixWidthTable create: #{ filename }"
|
9
9
|
@value_size = value_size
|
10
10
|
@range = range
|
11
11
|
@record_size = @value_size + (@range ? 16 : 8)
|
12
12
|
@write = true
|
13
13
|
|
14
|
-
if %w(memory stringio).include?
|
14
|
+
if %w(memory stringio).include?(filename.to_s.downcase)
|
15
15
|
@filename = :memory
|
16
16
|
@file = StringIO.new
|
17
17
|
else
|
18
18
|
FileUtils.rm @filename if File.exist? @filename
|
19
19
|
FileUtils.mkdir_p File.dirname(@filename) unless File.exist? @filename
|
20
|
-
|
20
|
+
#@file = File.open(@filename, 'wb')
|
21
|
+
@file = File.open(@filename, 'w:ASCII-8BIT')
|
21
22
|
end
|
22
23
|
|
23
24
|
@file.write [value_size].pack("L")
|
@@ -25,9 +26,9 @@ class FixWidthTable
|
|
25
26
|
|
26
27
|
@size = 0
|
27
28
|
else
|
28
|
-
Log.debug "FixWidthTable up-to-date: #{ filename }"
|
29
|
+
Log.debug "FixWidthTable up-to-date: #{ filename } - (in_memory:#{in_memory})"
|
29
30
|
if in_memory
|
30
|
-
@file =
|
31
|
+
@file = Open.open(@filename, :mode => 'r:ASCII-ASCII'){|f| StringIO.new f.read}
|
31
32
|
else
|
32
33
|
@file = File.open(@filename, 'r:ASCII-8BIT')
|
33
34
|
end
|
@@ -5,14 +5,15 @@ class KnowledgeBase
|
|
5
5
|
|
6
6
|
def select_entities(name, entities, options = {})
|
7
7
|
index = get_index(name, options)
|
8
|
+
|
8
9
|
source_field = index.source_field
|
9
10
|
target_field = index.target_field
|
10
11
|
|
11
12
|
source_type = Entity.formats[source_field]
|
12
13
|
target_type = Entity.formats[target_field]
|
13
14
|
|
14
|
-
source_entities = entities[:source] || entities[source_field] || entities[Entity.formats[source_field].to_s]
|
15
|
-
target_entities = entities[:target] || entities[target_field] || entities[Entity.formats[target_field].to_s]
|
15
|
+
source_entities = entities[:source] || entities[source_field] || entities[Entity.formats[source_field].to_s] || entities[:both]
|
16
|
+
target_entities = entities[:target] || entities[target_field] || entities[Entity.formats[target_field].to_s] || entities[:both]
|
16
17
|
|
17
18
|
[source_entities, target_entities]
|
18
19
|
end
|
@@ -10,6 +10,7 @@ class KnowledgeBase
|
|
10
10
|
|
11
11
|
def subset(name, entities, options = {}, &block)
|
12
12
|
entities, options = options, entities if entities.nil? and Hash === options
|
13
|
+
|
13
14
|
entities = case entities
|
14
15
|
when :all
|
15
16
|
{:target => :all, :source => :all}
|
@@ -23,7 +24,15 @@ class KnowledgeBase
|
|
23
24
|
raise "Entities are not a Hash or an AnnotatedArray: #{Misc.fingerprint entities}"
|
24
25
|
end
|
25
26
|
|
27
|
+
identify, identify_source, identify_target = entities.merge(options || {}).values_at :identify, :identify_source, :identify_target
|
28
|
+
|
26
29
|
source, target = select_entities(name, entities, options)
|
30
|
+
|
31
|
+
source = identify_source(name, source) if identify_source
|
32
|
+
target = identify_target(name, target) if identify_target
|
33
|
+
|
34
|
+
source = identify(name, source) if identify && !identify_source
|
35
|
+
target = identify(name, target) if identify && !identify_target
|
27
36
|
|
28
37
|
return [] if source.nil? or target.nil?
|
29
38
|
return [] if Array === target and target.empty?
|
data/lib/rbbt/persist.rb
CHANGED
@@ -407,7 +407,6 @@ module Persist
|
|
407
407
|
repo.read_and_close do
|
408
408
|
keys = repo.range subkey + 0.chr, true, subkey + 254.chr, true
|
409
409
|
end
|
410
|
-
repo.close
|
411
410
|
else
|
412
411
|
repo.read_and_close do
|
413
412
|
keys = repo.range subkey + 0.chr, true, subkey + 254.chr, true
|
@@ -461,13 +460,13 @@ module Persist
|
|
461
460
|
entities.each_with_index do |e,i|
|
462
461
|
next if e.nil?
|
463
462
|
tsv_values = e.tsv_values("literal", "annotation_types", "JSON")
|
464
|
-
repo[subkey +
|
463
|
+
repo[subkey + "ANNOTATED_DOUBLE_ARRAY:" << i.to_s] = tsv_values
|
465
464
|
end
|
466
465
|
else
|
467
466
|
entities.each_with_index do |e,i|
|
468
467
|
next if e.nil?
|
469
468
|
tsv_values = e.tsv_values("literal", "annotation_types", "JSON")
|
470
|
-
repo[subkey +
|
469
|
+
repo[subkey + i.to_s] = tsv_values
|
471
470
|
end
|
472
471
|
end
|
473
472
|
end
|
data/lib/rbbt/persist/tsv.rb
CHANGED
@@ -148,13 +148,13 @@ module Persist
|
|
148
148
|
|
149
149
|
data.write_and_read do
|
150
150
|
yield data
|
151
|
-
end
|
152
151
|
|
153
|
-
|
154
|
-
|
155
|
-
|
152
|
+
FileUtils.mv data.persistence_path, path if File.exist? data.persistence_path and not File.exist? path
|
153
|
+
tsv = CONNECTIONS[path] = CONNECTIONS.delete tmp_path
|
154
|
+
tsv.persistence_path = path
|
156
155
|
|
157
|
-
|
156
|
+
tsv.fix_io if tsv.respond_to? :fix_io
|
157
|
+
end
|
158
158
|
|
159
159
|
data
|
160
160
|
rescue Exception
|
@@ -9,43 +9,40 @@ module Persist
|
|
9
9
|
@mutex ||= Mutex.new
|
10
10
|
end
|
11
11
|
|
12
|
-
def prefix(key)
|
13
|
-
range(key, 1, key + MAX_CHAR, 1)
|
14
|
-
end
|
15
|
-
|
16
|
-
def get_prefix(key)
|
17
|
-
keys = prefix(key)
|
18
|
-
select(:key => keys)
|
19
|
-
end
|
20
|
-
|
21
12
|
def closed?
|
22
13
|
@closed
|
23
14
|
end
|
24
15
|
|
25
|
-
def close
|
26
|
-
@closed = true
|
27
|
-
super
|
28
|
-
self
|
29
|
-
end
|
30
|
-
|
31
16
|
def write?
|
32
17
|
@writable
|
33
18
|
end
|
34
19
|
|
35
20
|
def read?
|
36
|
-
! write?
|
21
|
+
! (write? || closed?)
|
37
22
|
end
|
38
23
|
|
39
|
-
def
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
24
|
+
def write(*args)
|
25
|
+
begin
|
26
|
+
super(*args)
|
27
|
+
@writable = true
|
28
|
+
rescue NoMethodError
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def close(*args)
|
33
|
+
begin
|
34
|
+
super(*args)
|
35
|
+
@closed = true
|
36
|
+
rescue NoMethodError
|
37
|
+
end
|
38
|
+
self
|
39
|
+
end
|
40
|
+
|
41
|
+
def read(*args)
|
42
|
+
begin
|
43
|
+
super(*args)
|
44
|
+
rescue NoMethodError
|
47
45
|
end
|
48
|
-
res
|
49
46
|
end
|
50
47
|
|
51
48
|
def delete(key)
|
@@ -53,34 +50,59 @@ module Persist
|
|
53
50
|
end
|
54
51
|
|
55
52
|
def lock
|
53
|
+
return yield if @locked
|
56
54
|
lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
|
55
|
+
Log.stack caller if $LOG
|
57
56
|
Misc.lock(lock_filename) do
|
58
|
-
|
57
|
+
begin
|
58
|
+
@locked = true
|
59
|
+
yield
|
60
|
+
ensure
|
61
|
+
@locked = false
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def lock_and_close
|
67
|
+
lock do
|
68
|
+
begin
|
69
|
+
yield
|
70
|
+
ensure
|
71
|
+
close
|
72
|
+
end
|
59
73
|
end
|
60
74
|
end
|
61
75
|
|
62
76
|
def write_and_read
|
63
|
-
|
64
|
-
|
65
|
-
|
77
|
+
if write?
|
78
|
+
begin
|
79
|
+
return yield
|
80
|
+
ensure
|
81
|
+
read
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
66
85
|
lock do
|
67
|
-
write
|
86
|
+
write(true) if closed? || !write?
|
68
87
|
begin
|
69
88
|
yield
|
70
89
|
ensure
|
71
90
|
read
|
72
91
|
end
|
73
92
|
end
|
74
|
-
#end
|
75
|
-
#end
|
76
93
|
end
|
77
94
|
|
78
95
|
def write_and_close
|
79
|
-
|
80
|
-
|
81
|
-
|
96
|
+
if write?
|
97
|
+
begin
|
98
|
+
return yield
|
99
|
+
ensure
|
100
|
+
close unless @locked
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
82
104
|
lock do
|
83
|
-
write
|
105
|
+
write(true) if closed? || ! write?
|
84
106
|
res = begin
|
85
107
|
yield
|
86
108
|
rescue Exception
|
@@ -91,11 +113,18 @@ module Persist
|
|
91
113
|
end
|
92
114
|
res
|
93
115
|
end
|
94
|
-
#end
|
95
|
-
#end
|
96
116
|
end
|
97
117
|
|
98
118
|
def read_and_close
|
119
|
+
#return yield if @locked
|
120
|
+
if read? || write?
|
121
|
+
begin
|
122
|
+
return yield
|
123
|
+
ensure
|
124
|
+
close unless @locked
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
99
128
|
lock do
|
100
129
|
read true if closed? || ! read?
|
101
130
|
begin
|
@@ -106,21 +135,85 @@ module Persist
|
|
106
135
|
end
|
107
136
|
end
|
108
137
|
|
109
|
-
|
110
138
|
def merge!(hash)
|
111
139
|
hash.each do |key,values|
|
112
140
|
self[key] = values
|
113
141
|
end
|
114
142
|
end
|
115
143
|
|
116
|
-
|
117
144
|
def range(*args)
|
118
|
-
|
145
|
+
self.read_and_close do
|
146
|
+
super(*args)
|
147
|
+
end
|
119
148
|
end
|
120
149
|
|
121
150
|
def include?(*args)
|
122
|
-
|
123
|
-
|
151
|
+
self.read_and_close do
|
152
|
+
super(*args) #- TSV::ENTRY_KEYS.to_a
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def [](*args)
|
157
|
+
self.read_and_close do
|
158
|
+
super(*args) #- TSV::ENTRY_KEYS.to_a
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
def []=(*args)
|
163
|
+
self.write_and_close do
|
164
|
+
super(*args) #- TSV::ENTRY_KEYS.to_a
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
def keys(*args)
|
169
|
+
self.read_and_close do
|
170
|
+
super(*args)
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
|
175
|
+
def prefix(key)
|
176
|
+
self.read_and_close do
|
177
|
+
range(key, 1, key + MAX_CHAR, 1)
|
178
|
+
end
|
179
|
+
end
|
180
|
+
|
181
|
+
def get_prefix(key)
|
182
|
+
keys = prefix(key)
|
183
|
+
select(:key => keys)
|
184
|
+
end
|
185
|
+
|
186
|
+
|
187
|
+
def size(*args)
|
188
|
+
self.read_and_close do
|
189
|
+
super(*args)
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
def each(*args, &block)
|
194
|
+
self.read_and_close do
|
195
|
+
super(*args, &block)
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
def collect
|
200
|
+
res = []
|
201
|
+
each do |key, value|
|
202
|
+
res << if block_given?
|
203
|
+
yield key, value
|
204
|
+
else
|
205
|
+
[key, value]
|
206
|
+
end
|
207
|
+
end
|
208
|
+
res
|
209
|
+
end
|
210
|
+
|
211
|
+
def values_at(*keys)
|
212
|
+
self.read_and_close do
|
213
|
+
keys.collect do |k|
|
214
|
+
self[k]
|
215
|
+
end
|
216
|
+
end
|
124
217
|
end
|
125
218
|
end
|
126
219
|
end
|
@@ -16,7 +16,7 @@ module Persist
|
|
16
16
|
end
|
17
17
|
|
18
18
|
dir = File.dirname(File.expand_path(path))
|
19
|
-
|
19
|
+
File.mkdir(dir) unless File.exists?(dir)
|
20
20
|
|
21
21
|
tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
|
22
22
|
tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
|
@@ -87,13 +87,15 @@ module Persist
|
|
87
87
|
def self.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB)
|
88
88
|
write = true unless File.exist? path
|
89
89
|
|
90
|
-
FileUtils.mkdir_p File.dirname(path) unless
|
90
|
+
FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))
|
91
91
|
|
92
92
|
database = Persist::TCAdapter.open(path, write, tokyocabinet_class)
|
93
93
|
|
94
94
|
unless serializer == :clean
|
95
95
|
TSV.setup database
|
96
|
-
database.
|
96
|
+
database.write_and_read do
|
97
|
+
database.serializer = serializer
|
98
|
+
end if serializer && database.serializer != serializer
|
97
99
|
end
|
98
100
|
|
99
101
|
database
|