rbbt-util 4.3.0 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +11 -6
- data/bin/rbbt_exec.rb +9 -1
- data/lib/rbbt/annotations.rb +78 -22
- data/lib/rbbt/persist.rb +14 -6
- data/lib/rbbt/persist/tsv.rb +10 -4
- data/lib/rbbt/resource.rb +1 -0
- data/lib/rbbt/resource/path.rb +43 -2
- data/lib/rbbt/resource/util.rb +3 -3
- data/lib/rbbt/tsv.rb +8 -2
- data/lib/rbbt/tsv/accessor.rb +29 -28
- data/lib/rbbt/tsv/attach.rb +8 -3
- data/lib/rbbt/tsv/attach/util.rb +3 -0
- data/lib/rbbt/tsv/excel.rb +91 -0
- data/lib/rbbt/tsv/filter.rb +17 -7
- data/lib/rbbt/tsv/manipulate.rb +26 -11
- data/lib/rbbt/tsv/parser.rb +3 -3
- data/lib/rbbt/tsv/util.rb +6 -29
- data/lib/rbbt/util/R.rb +27 -2
- data/lib/rbbt/util/chain_methods.rb +1 -2
- data/lib/rbbt/util/misc.rb +142 -13
- data/lib/rbbt/util/named_array.rb +5 -5
- data/lib/rbbt/util/open.rb +8 -4
- data/lib/rbbt/workflow.rb +5 -40
- data/lib/rbbt/workflow/accessor.rb +11 -0
- data/lib/rbbt/workflow/annotate.rb +8 -2
- data/lib/rbbt/workflow/step.rb +21 -2
- data/lib/rbbt/workflow/task.rb +3 -3
- data/share/lib/R/util.R +48 -15
- data/test/rbbt/resource/test_path.rb +15 -0
- data/test/rbbt/test_annotations.rb +11 -1
- data/test/rbbt/test_tsv.rb +15 -2
- data/test/rbbt/tsv/test_index.rb +27 -24
- data/test/rbbt/tsv/test_util.rb +1 -0
- data/test/rbbt/util/test_misc.rb +20 -0
- data/test/rbbt/util/test_open.rb +3 -5
- data/test/rbbt/workflow/test_step.rb +2 -2
- metadata +24 -9
data/README.rdoc
CHANGED
@@ -4,7 +4,8 @@
|
|
4
4
|
|
5
5
|
:script
|
6
6
|
|
7
|
-
|
7
|
+
curl -L https://get.rvm.io | bash -s stable --ruby
|
8
|
+
|
8
9
|
cd
|
9
10
|
cat .bash_profile > tmp.bash_profile
|
10
11
|
echo '[[ -s "$HOME/.rvm/scripts/rvm" ]] && . "$HOME/.rvm/scripts/rvm"' >> tmp.bash_profile
|
@@ -42,7 +43,7 @@
|
|
42
43
|
|
43
44
|
:script
|
44
45
|
|
45
|
-
gem install rbbt-util rbbt-sources rbbt-text rbbt-phgx
|
46
|
+
gem install rbbt-util rbbt-sources rbbt-text rbbt-phgx rbbt-entities rbbt-views
|
46
47
|
|
47
48
|
3.bis - Or install github repos and make ruby use them
|
48
49
|
|
@@ -50,10 +51,13 @@
|
|
50
51
|
|
51
52
|
cd
|
52
53
|
cd git/
|
53
|
-
git clone git
|
54
|
-
git clone git
|
55
|
-
git clone git
|
56
|
-
git clone git
|
54
|
+
git clone git@github.com:mikisvaz/rbbt-util.git
|
55
|
+
git clone git@github.com:mikisvaz/rbbt-sources.git
|
56
|
+
git clone git@github.com:mikisvaz/rbbt-text.git
|
57
|
+
git clone git@github.com:mikisvaz/rbbt-phgx.git
|
58
|
+
git clone git@github.com:mikisvaz/rbbt-entities.git
|
59
|
+
git clone git@github.com:mikisvaz/rbbt-dm.git
|
60
|
+
git clone git@github.com:mikisvaz/rbbt-views.git
|
57
61
|
alias druby="env RBBT_LOG=0 ruby $(for d in $HOME/git/rbbt-*;do echo -n "-I$d/lib ";done)"
|
58
62
|
alias drake="env RBBT_LOG=0 rake $(for d in $HOME/git/rbbt-*;do echo -n "-I$d/lib ";done)"
|
59
63
|
|
@@ -67,6 +71,7 @@
|
|
67
71
|
|
68
72
|
:script
|
69
73
|
|
74
|
+
mkdir -p ~/.rbbt/etc
|
70
75
|
mkdir -p ~/git/workflows
|
71
76
|
echo "~/git/workflows/" > ~/.rbbt/etc/workflow_dir
|
72
77
|
|
data/bin/rbbt_exec.rb
CHANGED
@@ -12,9 +12,17 @@ code = case
|
|
12
12
|
Open.read(code_file)
|
13
13
|
end
|
14
14
|
|
15
|
-
|
15
|
+
begin
|
16
|
+
data = eval code
|
17
|
+
rescue Exception
|
18
|
+
puts "#:rbbt_exec Error"
|
19
|
+
puts $!.message
|
20
|
+
puts $!.backtrace * "\n"
|
21
|
+
exit(-1)
|
22
|
+
end
|
16
23
|
|
17
24
|
data = data.to_s(:sort, true) if TSV === data
|
25
|
+
data = data * "\n" if Array === data
|
18
26
|
|
19
27
|
case
|
20
28
|
when (output.nil? or output == '-')
|
data/lib/rbbt/annotations.rb
CHANGED
@@ -7,6 +7,7 @@ module Annotated
|
|
7
7
|
attr_accessor :context
|
8
8
|
attr_accessor :container
|
9
9
|
attr_accessor :container_index
|
10
|
+
attr_accessor :id
|
10
11
|
|
11
12
|
def self.extended(base)
|
12
13
|
base.annotation_types ||= []
|
@@ -29,12 +30,12 @@ module Annotated
|
|
29
30
|
end
|
30
31
|
|
31
32
|
def id
|
32
|
-
Misc.hash2md5
|
33
|
+
@id ||= self.respond_to?(:annotation_id) ? annotation_id : Misc.hash2md5(info.merge(:self => self))
|
33
34
|
end
|
34
35
|
|
35
36
|
def self.load(object, info)
|
36
37
|
annotation_types = info[:annotation_types] || []
|
37
|
-
annotation_types = annotation_types.split("
|
38
|
+
annotation_types = annotation_types.split("|") if String === annotation_types
|
38
39
|
|
39
40
|
return object if annotation_types.nil? or annotation_types.empty?
|
40
41
|
|
@@ -43,6 +44,7 @@ module Annotated
|
|
43
44
|
mod.setup(object, *info.values_at(*mod.all_annotations))
|
44
45
|
end
|
45
46
|
|
47
|
+
object.id = info[:entity_id] if info.include? :entity_id
|
46
48
|
object
|
47
49
|
end
|
48
50
|
|
@@ -61,13 +63,15 @@ module Annotated
|
|
61
63
|
when field == "JSON"
|
62
64
|
info.to_json
|
63
65
|
when field == "annotation_types"
|
64
|
-
annotation_types.collect{|t| t.to_s} * "
|
66
|
+
annotation_types.collect{|t| t.to_s} * "|"
|
65
67
|
when field == "literal"
|
66
68
|
(Array === self ? "Array:" << self * "|" : self).gsub(/\n|\t/, ' ')
|
67
69
|
when info.include?(field.to_sym)
|
68
|
-
info.delete(field.to_sym)
|
70
|
+
res = info.delete(field.to_sym)
|
71
|
+
Array === res ? "Array:" << res * "|" : res
|
69
72
|
when self.respond_to?(field)
|
70
|
-
self.send(field)
|
73
|
+
res = self.send(field)
|
74
|
+
Array === res ? "Array:"<< res * "|" : res
|
71
75
|
end
|
72
76
|
end
|
73
77
|
|
@@ -89,7 +93,7 @@ module Annotated
|
|
89
93
|
|
90
94
|
object = object["Array:".length..-1].split("|") if object =~ /^Array:/
|
91
95
|
|
92
|
-
if Array === values
|
96
|
+
if Array === values.first
|
93
97
|
Misc.zip_fields(values).collect do |list|
|
94
98
|
fields.each_with_index do |field,i|
|
95
99
|
if field == "JSON"
|
@@ -116,6 +120,19 @@ module Annotated
|
|
116
120
|
self.load(object, info)
|
117
121
|
end
|
118
122
|
|
123
|
+
def self.json(annotations, literal = false)
|
124
|
+
annotations = [annotations] unless Array === annotations
|
125
|
+
hash = {}
|
126
|
+
annotations.each do |annotation|
|
127
|
+
if literal
|
128
|
+
hash[annotation.id] = annotation.info.merge(:literal => annotation)
|
129
|
+
else
|
130
|
+
hash[annotation.id] = annotation.info
|
131
|
+
end
|
132
|
+
end
|
133
|
+
hash.to_json
|
134
|
+
end
|
135
|
+
|
119
136
|
def self.tsv(annotations, *fields)
|
120
137
|
return nil if annotations.nil?
|
121
138
|
fields = case
|
@@ -124,8 +141,15 @@ module Annotated
|
|
124
141
|
fields << :annotation_types
|
125
142
|
when (fields == [:literal] and not annotations.empty?)
|
126
143
|
fields << :literal
|
144
|
+
when (fields == [:all] and Annotated === annotations)
|
145
|
+
fields = [:annotation_types] + annotations.annotations
|
146
|
+
fields << :literal
|
127
147
|
when (fields == [:all] and not annotations.empty?)
|
128
|
-
|
148
|
+
raise "Input array must be annotated or its elements must be" if not Annotated === annotations.compact.first and not Array === annotations.compact.first
|
149
|
+
raise "Input array must be annotated or its elements must be. No duble arrays of singly annotated entities." if not Annotated === annotations.compact.first and Array === annotations.compact.first
|
150
|
+
fields = [:annotation_types] + (Annotated === annotations ?
|
151
|
+
annotations.annotations :
|
152
|
+
annotations.compact.first.annotations)
|
129
153
|
fields << :literal
|
130
154
|
when annotations.empty?
|
131
155
|
[:annotation_types, :literal]
|
@@ -137,7 +161,7 @@ module Annotated
|
|
137
161
|
|
138
162
|
case
|
139
163
|
when (Annotated === annotations and not (AnnotatedArray === annotations and annotations.double_array))
|
140
|
-
tsv = TSV.setup({}, :key_field => "
|
164
|
+
tsv = TSV.setup({}, :key_field => "List", :fields => fields, :type => :list, :unnamed => true)
|
141
165
|
tsv[annotations.id] = annotations.tsv_values(*fields)
|
142
166
|
when Array === annotations
|
143
167
|
tsv = TSV.setup({}, :key_field => "ID", :fields => fields, :type => :list, :unnamed => true)
|
@@ -158,7 +182,7 @@ module Annotated
|
|
158
182
|
end
|
159
183
|
|
160
184
|
case tsv.key_field
|
161
|
-
when "
|
185
|
+
when "List"
|
162
186
|
annotated_entities.first
|
163
187
|
else
|
164
188
|
annotated_entities
|
@@ -251,6 +275,7 @@ module Annotation
|
|
251
275
|
|
252
276
|
inputs = Misc.positional2hash(all_annotations, *values)
|
253
277
|
inputs.each do |name, value|
|
278
|
+
value = value.split("|") if String === value and value.index "|"
|
254
279
|
object.send(name.to_s + '=', value)
|
255
280
|
end
|
256
281
|
|
@@ -282,9 +307,11 @@ module AnnotatedArray
|
|
282
307
|
annotation_types.each do |mod|
|
283
308
|
mod.setup(value, *info.values_at(*mod.all_annotations))
|
284
309
|
end
|
285
|
-
|
286
|
-
|
287
|
-
|
310
|
+
if Annotated === value
|
311
|
+
value.context = self.context
|
312
|
+
value.container = self
|
313
|
+
value.container_index = pos
|
314
|
+
end
|
288
315
|
value
|
289
316
|
end
|
290
317
|
|
@@ -295,9 +322,11 @@ module AnnotatedArray
|
|
295
322
|
annotation_types.each do |mod|
|
296
323
|
mod.setup(value, info)
|
297
324
|
end
|
298
|
-
|
299
|
-
|
300
|
-
|
325
|
+
if Annotated === value
|
326
|
+
value.context = self.context
|
327
|
+
value.container = self
|
328
|
+
value.container_index = i
|
329
|
+
end
|
301
330
|
i += 1
|
302
331
|
yield value
|
303
332
|
end
|
@@ -319,18 +348,25 @@ module AnnotatedArray
|
|
319
348
|
res
|
320
349
|
end
|
321
350
|
|
322
|
-
def annotated_array_select
|
351
|
+
def annotated_array_select(method = nil, *args)
|
323
352
|
res = []
|
324
|
-
|
325
|
-
res
|
353
|
+
if method
|
354
|
+
res = self.zip(self.send(method, *args)).select{|e,result| result}.collect{|element,r| element}
|
355
|
+
else
|
356
|
+
annotated_array_each do |value|
|
357
|
+
res << value if yield(value)
|
358
|
+
end
|
326
359
|
end
|
327
360
|
|
328
361
|
annotation_types.each do |mod|
|
329
362
|
mod.setup(res, *info.values_at(*mod.annotations))
|
330
363
|
end
|
364
|
+
|
331
365
|
res.context = self.context
|
332
366
|
res.container = self.container
|
333
367
|
|
368
|
+
res.extend AnnotatedArray if AnnotatedArray === self
|
369
|
+
|
334
370
|
res
|
335
371
|
end
|
336
372
|
|
@@ -346,16 +382,23 @@ module AnnotatedArray
|
|
346
382
|
res.context = self.context
|
347
383
|
res.container = self.container
|
348
384
|
|
385
|
+
res.extend AnnotatedArray if AnnotatedArray === self
|
386
|
+
|
349
387
|
res
|
350
388
|
end
|
351
389
|
|
352
390
|
def annotated_array_subset(list)
|
353
391
|
value = (self & list)
|
392
|
+
|
354
393
|
annotation_types.each do |mod|
|
355
394
|
mod.setup(value, *info.values_at(*mod.all_annotations))
|
356
395
|
end
|
396
|
+
|
357
397
|
value.context = self.context
|
358
398
|
value.container = self.container
|
399
|
+
|
400
|
+
value.extend AnnotatedArray if AnnotatedArray === self
|
401
|
+
|
359
402
|
value
|
360
403
|
end
|
361
404
|
|
@@ -369,6 +412,8 @@ module AnnotatedArray
|
|
369
412
|
value.context = self.context
|
370
413
|
value.container = self.container
|
371
414
|
|
415
|
+
value.extend AnnotatedArray if AnnotatedArray === self
|
416
|
+
|
372
417
|
value
|
373
418
|
end
|
374
419
|
|
@@ -382,6 +427,8 @@ module AnnotatedArray
|
|
382
427
|
value.context = self.context
|
383
428
|
value.container = self.container
|
384
429
|
|
430
|
+
value.extend AnnotatedArray if AnnotatedArray === self
|
431
|
+
|
385
432
|
value
|
386
433
|
end
|
387
434
|
|
@@ -395,6 +442,8 @@ module AnnotatedArray
|
|
395
442
|
value.context = self.context
|
396
443
|
value.container = self.container
|
397
444
|
|
445
|
+
value.extend AnnotatedArray if AnnotatedArray === self
|
446
|
+
|
398
447
|
value
|
399
448
|
end
|
400
449
|
|
@@ -408,6 +457,8 @@ module AnnotatedArray
|
|
408
457
|
value.context = self.context
|
409
458
|
value.container = self.container
|
410
459
|
|
460
|
+
value.extend AnnotatedArray if AnnotatedArray === self
|
461
|
+
|
411
462
|
value
|
412
463
|
end
|
413
464
|
|
@@ -421,6 +472,8 @@ module AnnotatedArray
|
|
421
472
|
value.context = self.context
|
422
473
|
value.container = self.container
|
423
474
|
|
475
|
+
value.extend AnnotatedArray if AnnotatedArray === self
|
476
|
+
|
424
477
|
value
|
425
478
|
end
|
426
479
|
|
@@ -434,18 +487,19 @@ module AnnotatedArray
|
|
434
487
|
value.context = self.context
|
435
488
|
value.container = self.container
|
436
489
|
|
490
|
+
value.extend AnnotatedArray if AnnotatedArray === self
|
491
|
+
|
437
492
|
value
|
438
493
|
end
|
439
494
|
|
440
|
-
def annotated_array_sort
|
441
|
-
value = self.
|
495
|
+
def annotated_array_sort(&block)
|
496
|
+
value = self.collect.sort(&block).collect{|value| value.respond_to?(:clean_annotations) ? value.clean_annotations.dup : value.dup }
|
442
497
|
|
443
498
|
annotation_types.each do |mod|
|
444
499
|
mod.setup(value, *info.values_at(*mod.annotations))
|
445
500
|
end
|
446
501
|
|
447
|
-
value.
|
448
|
-
value.container = self.container
|
502
|
+
value.extend AnnotatedArray if AnnotatedArray === self
|
449
503
|
|
450
504
|
value
|
451
505
|
end
|
@@ -466,6 +520,8 @@ module AnnotatedArray
|
|
466
520
|
value << e if method.shift
|
467
521
|
end
|
468
522
|
|
523
|
+
value.extend AnnotatedArray if AnnotatedArray === self
|
524
|
+
|
469
525
|
value
|
470
526
|
end
|
471
527
|
|
data/lib/rbbt/persist.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
require 'digest/md5'
|
2
|
-
|
3
1
|
require 'rbbt/util/misc'
|
4
2
|
require 'rbbt/util/open'
|
5
3
|
|
@@ -91,6 +89,11 @@ module Persist
|
|
91
89
|
FixWidthTable.get(path)
|
92
90
|
when :string, :text
|
93
91
|
Open.read(path)
|
92
|
+
when :binary
|
93
|
+
f = File.open(path, 'rb')
|
94
|
+
res = f.read
|
95
|
+
f.close
|
96
|
+
res
|
94
97
|
when :array
|
95
98
|
res = Open.read(path).split("\n", -1)
|
96
99
|
res.pop
|
@@ -128,11 +131,16 @@ module Persist
|
|
128
131
|
Open.write(path, Annotated.tsv(content, :all).to_s)
|
129
132
|
when :string, :text
|
130
133
|
Open.write(path, content)
|
134
|
+
when :binary
|
135
|
+
f = File.open(path, 'wb')
|
136
|
+
f.puts content
|
137
|
+
f.close
|
138
|
+
content
|
131
139
|
when :array
|
132
|
-
if content.
|
133
|
-
Open.write(path, content * "\n" + "\n")
|
134
|
-
else
|
140
|
+
if content.empty?
|
135
141
|
Open.write(path, "")
|
142
|
+
else
|
143
|
+
Open.write(path, content * "\n" + "\n")
|
136
144
|
end
|
137
145
|
when :marshal_tsv
|
138
146
|
Open.write(path, Marshal.dump(content.dup))
|
@@ -185,7 +193,7 @@ module Persist
|
|
185
193
|
[]
|
186
194
|
when (keys.length == 1 and keys.first =~ /:SINGLE$/)
|
187
195
|
key = keys.first
|
188
|
-
values = repo.
|
196
|
+
values = repo.read_and_close do
|
189
197
|
repo[key]
|
190
198
|
end
|
191
199
|
Annotated.load_tsv_values(key, values, "literal", "annotation_types", "JSON")
|
data/lib/rbbt/persist/tsv.rb
CHANGED
@@ -83,15 +83,21 @@ module Persist
|
|
83
83
|
|
84
84
|
def write_and_close
|
85
85
|
write if @closed or not write?
|
86
|
-
res =
|
87
|
-
|
86
|
+
res = begin
|
87
|
+
yield
|
88
|
+
ensure
|
89
|
+
close
|
90
|
+
end
|
88
91
|
res
|
89
92
|
end
|
90
93
|
|
91
94
|
def read_and_close
|
92
95
|
read if @closed or write?
|
93
|
-
res =
|
94
|
-
|
96
|
+
res = begin
|
97
|
+
yield
|
98
|
+
ensure
|
99
|
+
close
|
100
|
+
end
|
95
101
|
res
|
96
102
|
end
|
97
103
|
|
data/lib/rbbt/resource.rb
CHANGED
data/lib/rbbt/resource/path.rb
CHANGED
@@ -36,6 +36,10 @@ module Path
|
|
36
36
|
Path.setup File.dirname(self), @pkgdir, @resource
|
37
37
|
end
|
38
38
|
|
39
|
+
def glob(pattern = '*')
|
40
|
+
Dir.glob(File.join(self, pattern))
|
41
|
+
end
|
42
|
+
|
39
43
|
def path_get_brackets(name)
|
40
44
|
join name
|
41
45
|
end
|
@@ -128,8 +132,8 @@ module Path
|
|
128
132
|
Open.read(self.produce.find)
|
129
133
|
end
|
130
134
|
|
131
|
-
def open
|
132
|
-
Open.open(self.produce.find)
|
135
|
+
def open(options = {})
|
136
|
+
Open.open(self.produce.find, options)
|
133
137
|
end
|
134
138
|
|
135
139
|
def to_s
|
@@ -191,4 +195,41 @@ module Path
|
|
191
195
|
[identifier_file_path]
|
192
196
|
end
|
193
197
|
end
|
198
|
+
|
199
|
+
def set_extension(new_extension = nil)
|
200
|
+
new_path = self.sub(/\.[^\.\/]+$/, "." << new_extension.to_s)
|
201
|
+
Path.setup new_path, @pkgdir, @resource
|
202
|
+
end
|
203
|
+
|
204
|
+
def doc_file(relative_to = 'lib')
|
205
|
+
if located?
|
206
|
+
lib_dir = Path.caller_lib_dir(self, relative_to)
|
207
|
+
relative_file = File.join( 'doc', self.sub(lib_dir,''))
|
208
|
+
Path.setup File.join(lib_dir, relative_file) , @pkgdir, @resource
|
209
|
+
else
|
210
|
+
Path.setup File.join('doc', self) , @pkgdir, @resource
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
def source_for_doc_file(relative_to = 'lib')
|
215
|
+
if located?
|
216
|
+
lib_dir = Path.caller_lib_dir(Path.caller_lib_dir(self, 'doc'), relative_to)
|
217
|
+
relative_file = self.sub(/(.*\/)doc\//, '\1').sub(lib_dir + "/",'')
|
218
|
+
file = File.join(lib_dir, relative_file)
|
219
|
+
|
220
|
+
if not File.exists?(file)
|
221
|
+
file= Dir.glob(file.sub(/\.[^\.\/]+$/, '.*')).first
|
222
|
+
end
|
223
|
+
|
224
|
+
Path.setup file, @pkgdir, @resource
|
225
|
+
else
|
226
|
+
relative_file = self.sub(/^doc\//, '\1')
|
227
|
+
|
228
|
+
if not File.exists?(relative_file)
|
229
|
+
relative_file = Dir.glob(relative_file.sub(/\.[^\.\/]+$/, '.*')).first
|
230
|
+
end
|
231
|
+
|
232
|
+
Path.setup relative_file , @pkgdir, @resource
|
233
|
+
end
|
234
|
+
end
|
194
235
|
end
|