rbbt-util 3.0.3 → 3.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/rbbt_Rutil.rb +4 -0
- data/bin/rbbt_exec.rb +33 -0
- data/lib/rbbt/util/R.rb +3 -2
- data/lib/rbbt/util/cmd.rb +13 -2
- data/lib/rbbt/util/fix_width_table.rb +2 -0
- data/lib/rbbt/util/misc.rb +8 -1
- data/lib/rbbt/util/open.rb +2 -2
- data/lib/rbbt/util/persistence.rb +27 -18
- data/lib/rbbt/util/resource.rb +3 -2
- data/lib/rbbt/util/task.rb +77 -5
- data/lib/rbbt/util/task/job.rb +20 -4
- data/lib/rbbt/util/tc_hash.rb +2 -1
- data/lib/rbbt/util/tsv.rb +59 -33
- data/lib/rbbt/util/tsv/accessor.rb +27 -2
- data/lib/rbbt/util/tsv/attach.rb +48 -121
- data/lib/rbbt/util/tsv/index.rb +4 -0
- data/lib/rbbt/util/tsv/manipulate.rb +25 -3
- data/lib/rbbt/util/tsv/misc.rb +31 -0
- data/lib/rbbt/util/tsv/parse.rb +27 -4
- data/lib/rbbt/util/tsv/resource.rb +6 -0
- data/lib/rbbt/util/workflow.rb +1 -1
- data/lib/rbbt/util/workflow/soap.rb +117 -0
- data/share/lib/R/util.R +52 -2
- data/test/rbbt/util/test_misc.rb +11 -11
- data/test/rbbt/util/test_persistence.rb +13 -0
- data/test/rbbt/util/test_tc_hash.rb +4 -2
- data/test/rbbt/util/test_tsv.rb +31 -4
- data/test/rbbt/util/test_workflow.rb +11 -3
- data/test/rbbt/util/tsv/test_attach.rb +35 -1
- data/test/rbbt/util/tsv/test_index.rb +1 -3
- metadata +12 -6
data/bin/rbbt_Rutil.rb
ADDED
data/bin/rbbt_exec.rb
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
code_file = ARGV[0]
|
4
|
+
output = ARGV[1]
|
5
|
+
|
6
|
+
require 'rbbt-util'
|
7
|
+
|
8
|
+
code = case
|
9
|
+
when (code_file.nil? or code_file == '-')
|
10
|
+
STDIN.read
|
11
|
+
else
|
12
|
+
Open.read(code_file)
|
13
|
+
end
|
14
|
+
|
15
|
+
data = eval code
|
16
|
+
|
17
|
+
data = data.to_s(:sort, true) if TSV === data
|
18
|
+
|
19
|
+
case
|
20
|
+
when (output.nil? or output == '-')
|
21
|
+
puts data
|
22
|
+
when output == "file"
|
23
|
+
if Misc.filename? data
|
24
|
+
tmpfile = data
|
25
|
+
else
|
26
|
+
tmpfile = TmpFile.tmp_file
|
27
|
+
Open.write(tmpfile, data)
|
28
|
+
end
|
29
|
+
|
30
|
+
puts tmpfile
|
31
|
+
else
|
32
|
+
Open.write(output, data)
|
33
|
+
end
|
data/lib/rbbt/util/R.rb
CHANGED
@@ -25,7 +25,7 @@ module R
|
|
25
25
|
end
|
26
26
|
|
27
27
|
class TSV
|
28
|
-
def R(script)
|
28
|
+
def R(script, open_options = {})
|
29
29
|
TmpFile.with_file do |f|
|
30
30
|
Open.write(f, self.to_s)
|
31
31
|
Log.debug(R.run(
|
@@ -35,7 +35,8 @@ data = rbbt.tsv('#{f}');
|
|
35
35
|
rbbt.tsv.write('#{f}', data);
|
36
36
|
EOF
|
37
37
|
).read)
|
38
|
-
|
38
|
+
open_options = Misc.add_defaults open_options, :type => :list
|
39
|
+
TSV.new(f, open_options)
|
39
40
|
end
|
40
41
|
end
|
41
42
|
end
|
data/lib/rbbt/util/cmd.rb
CHANGED
@@ -29,16 +29,28 @@ module CMD
|
|
29
29
|
original_close
|
30
30
|
end
|
31
31
|
|
32
|
+
def force_close
|
33
|
+
if @pid
|
34
|
+
Log.debug "Forcing close by killing '#{@pid}'"
|
35
|
+
Process.kill("KILL", @pid)
|
36
|
+
Process.waitpid(@pid)
|
37
|
+
end
|
38
|
+
@post.call if @post
|
39
|
+
original_close
|
40
|
+
end
|
41
|
+
|
32
42
|
alias original_read read
|
33
43
|
def read
|
34
44
|
data = Misc.fixutf8(original_read)
|
35
45
|
self.close unless self.closed?
|
36
46
|
data
|
37
47
|
end
|
48
|
+
|
38
49
|
}
|
39
50
|
io
|
40
51
|
end
|
41
|
-
|
52
|
+
|
53
|
+
end
|
42
54
|
|
43
55
|
def self.process_cmd_options(options = {})
|
44
56
|
string = ""
|
@@ -109,7 +121,6 @@ module CMD
|
|
109
121
|
rescue Exception
|
110
122
|
raise CMDError, $!.message
|
111
123
|
end
|
112
|
-
|
113
124
|
}
|
114
125
|
sin.first.close
|
115
126
|
sout.last.close
|
data/lib/rbbt/util/misc.rb
CHANGED
@@ -33,6 +33,10 @@ end
|
|
33
33
|
module Misc
|
34
34
|
class FieldNotFoundError < StandardError;end
|
35
35
|
|
36
|
+
def self.filename?(filename)
|
37
|
+
String === filename and filename.length < 1024 and filename.index("\n").nil? and File.exists? filename
|
38
|
+
end
|
39
|
+
|
36
40
|
def self.lock(file, *args)
|
37
41
|
FileUtils.mkdir_p File.dirname(File.expand_path(file)) unless File.exists? File.dirname(File.expand_path(file))
|
38
42
|
lockfile = Lockfile.new file + '.lock'
|
@@ -162,12 +166,13 @@ module Misc
|
|
162
166
|
def self.hash2md5(hash)
|
163
167
|
o = {}
|
164
168
|
hash.keys.sort_by{|k| k.to_s}.each do |k|
|
169
|
+
next if k == :monitor or k == "monitor" or k == :in_situ_persistence or k == "in_situ_persistence"
|
165
170
|
v = hash[k]
|
166
171
|
case
|
167
172
|
when v.inspect =~ /:0x0/
|
168
173
|
o[k] = v.inspect.sub(/:0x[a-f0-9]+@/,'')
|
169
174
|
when Resource::Path === v
|
170
|
-
"" << String.new(v.to_s)
|
175
|
+
o[k] = "" << String.new(v.to_s)
|
171
176
|
else
|
172
177
|
o[k] = v
|
173
178
|
end
|
@@ -230,6 +235,7 @@ module Misc
|
|
230
235
|
File.open(path, 'w') do |f| end
|
231
236
|
end
|
232
237
|
rescue Interrupt
|
238
|
+
FileUtils.rm_f path
|
233
239
|
raise "Interrupted (Ctrl-c)"
|
234
240
|
rescue Exception
|
235
241
|
FileUtils.rm_f path
|
@@ -373,6 +379,7 @@ class NamedArray < Array
|
|
373
379
|
end
|
374
380
|
|
375
381
|
def zip_fields
|
382
|
+
return [] if self.empty?
|
376
383
|
zipped = self[0].zip(*self[1..-1])
|
377
384
|
zipped = zipped.collect{|v| NamedArray.name(v, fields)} if fields
|
378
385
|
zipped
|
data/lib/rbbt/util/open.rb
CHANGED
@@ -104,7 +104,7 @@ module Open
|
|
104
104
|
CMD.cmd("grep", "-E" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
|
105
105
|
end
|
106
106
|
else
|
107
|
-
CMD.cmd("grep '#{grep}' -", :in => stream, :pipe => true)
|
107
|
+
CMD.cmd("grep '#{grep}' -", :in => stream, :pipe => true, :post => proc{stream.force_close if stream.respond_to? :force_close})
|
108
108
|
end
|
109
109
|
end
|
110
110
|
|
@@ -122,7 +122,7 @@ module Open
|
|
122
122
|
if String === stream
|
123
123
|
Zlib::Inflate.inflate(stream)
|
124
124
|
else
|
125
|
-
CMD.cmd("gunzip", :pipe => true, :in => stream)
|
125
|
+
CMD.cmd("gunzip", :pipe => true, :in => stream, :post => proc{stream.force_close if stream.respond_to? :force_close})
|
126
126
|
end
|
127
127
|
end
|
128
128
|
|
@@ -23,7 +23,7 @@ module Persistence
|
|
23
23
|
def self.get_persistence_file(file, prefix, options = {})
|
24
24
|
persistence_dir = Misc.process_options options, :persistence_dir
|
25
25
|
persistence_dir ||= CACHEDIR
|
26
|
-
name = prefix.to_s << ":" << file.to_s << ":"
|
26
|
+
name = prefix.to_s.dup << ":" << file.to_s << ":"
|
27
27
|
|
28
28
|
options_md5 = Misc.hash2md5 options
|
29
29
|
File.join(persistence_dir, name.to_s.gsub(/\s/,'_').gsub(/\//,'>') + options_md5)
|
@@ -193,10 +193,10 @@ module Persistence
|
|
193
193
|
end
|
194
194
|
end
|
195
195
|
|
196
|
-
per.read
|
197
|
-
|
198
196
|
tsv = Object::TSV.new per
|
199
197
|
|
198
|
+
per.read
|
199
|
+
|
200
200
|
tsv
|
201
201
|
else
|
202
202
|
Log.debug "Loading #{ persistence_file }. Prefix = #{prefix}"
|
@@ -208,6 +208,8 @@ module Persistence
|
|
208
208
|
tsv.send "#{key}=".to_sym, per.send(key.to_sym)
|
209
209
|
end
|
210
210
|
end
|
211
|
+
|
212
|
+
per.read
|
211
213
|
|
212
214
|
tsv
|
213
215
|
end
|
@@ -228,28 +230,32 @@ module Persistence
|
|
228
230
|
|
229
231
|
serializer = tsv_serializer res, extra
|
230
232
|
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
233
|
+
per = nil
|
234
|
+
if not Persistence::TSV === res
|
235
|
+
begin
|
236
|
+
per = Persistence::TSV.get persistence_file, true, serializer
|
237
|
+
|
238
|
+
per.write
|
239
|
+
per.merge! res
|
240
|
+
Persistence::TSV::FIELD_INFO_ENTRIES.keys.each do |key|
|
241
|
+
if extra.include?(key.to_sym) and per.respond_to?(key.to_sym)
|
242
|
+
per.send "#{key}=".to_sym, extra[key.to_sym]
|
243
|
+
end
|
239
244
|
end
|
245
|
+
|
246
|
+
rescue Exception
|
247
|
+
per.close
|
248
|
+
raise $!
|
240
249
|
end
|
241
|
-
|
242
|
-
per
|
243
|
-
raise $!
|
250
|
+
else
|
251
|
+
per = res
|
244
252
|
end
|
245
253
|
|
246
|
-
per.read
|
247
|
-
|
248
254
|
[ per, extra ]
|
249
255
|
else
|
250
256
|
Log.debug "Loading #{ persistence_file }. Prefix = #{prefix}"
|
251
257
|
begin
|
252
|
-
per = Persistence::TSV.get persistence_file,
|
258
|
+
per = Persistence::TSV.get persistence_file, false, serializer
|
253
259
|
|
254
260
|
extra = {}
|
255
261
|
Persistence::TSV::FIELD_INFO_ENTRIES.keys.each do |key|
|
@@ -258,10 +264,13 @@ module Persistence
|
|
258
264
|
end
|
259
265
|
end
|
260
266
|
|
267
|
+
rescue Interrupt
|
268
|
+
raise "Interrupted"
|
261
269
|
rescue Exception
|
262
270
|
per.close
|
263
271
|
raise $!
|
264
272
|
end
|
273
|
+
|
265
274
|
[ per, extra ]
|
266
275
|
end
|
267
276
|
end
|
@@ -276,7 +285,7 @@ module Persistence
|
|
276
285
|
persistence_file ||= get_persistence_file(filename, prefix, options)
|
277
286
|
|
278
287
|
if persistence_update or not File.exists? persistence_file
|
279
|
-
Log.debug "Creating #{ persistence_file }. Prefix = #{prefix}"
|
288
|
+
Log.debug "Creating FWT #{ persistence_file }. Prefix = #{prefix}"
|
280
289
|
|
281
290
|
range = options[:range]
|
282
291
|
|
data/lib/rbbt/util/resource.rb
CHANGED
@@ -180,13 +180,14 @@ source "$INSTALL_HELPER_FILE"
|
|
180
180
|
end
|
181
181
|
|
182
182
|
def data_module(klass)
|
183
|
-
relative_to klass, "share/#{self.to_s.downcase}"
|
183
|
+
relative_to klass, "share/#{self.to_s.downcase}" unless klass == base
|
184
184
|
rakefile = klass.share.install[self.to_s].Rakefile
|
185
185
|
rakefile.lib_dir = Resource.caller_lib_dir
|
186
186
|
|
187
|
+
|
187
188
|
self[''].define_as_rake rakefile
|
188
189
|
self.namespace = base.to_s
|
189
|
-
self.lib_dir = caller_lib_dir
|
190
|
+
self.lib_dir = Resource.caller_lib_dir
|
190
191
|
end
|
191
192
|
|
192
193
|
module Path
|
data/lib/rbbt/util/task.rb
CHANGED
@@ -64,13 +64,16 @@ class Task
|
|
64
64
|
dependencies.each do |dependency|
|
65
65
|
case
|
66
66
|
when Proc === dependency
|
67
|
-
|
68
|
-
|
69
|
-
|
67
|
+
deps = dependency.call(jobname, run_options)
|
68
|
+
if Array === deps
|
69
|
+
previous_jobs.concat deps
|
70
|
+
else
|
71
|
+
previous_jobs << deps
|
72
|
+
end
|
70
73
|
when Task::Job === dependency
|
71
74
|
previous_jobs << dependency
|
72
|
-
when
|
73
|
-
previous_jobs <<
|
75
|
+
when Task === dependency
|
76
|
+
previous_jobs << dependency.job(jobname, *(args + [optional_args]))
|
74
77
|
else
|
75
78
|
required_files << dependency
|
76
79
|
end
|
@@ -97,4 +100,73 @@ class Task
|
|
97
100
|
job(*args).start
|
98
101
|
end
|
99
102
|
|
103
|
+
def option_info(option)
|
104
|
+
info = {}
|
105
|
+
info[:name] = option
|
106
|
+
info[:source] = name
|
107
|
+
info[:description] = option_descriptions[option] if option_descriptions and option_descriptions.include? option
|
108
|
+
info[:type] = option_types[option] if option_types and option_types.include? option
|
109
|
+
info[:default] = option_defaults[option] if option_defaults and option_defaults.include? option
|
110
|
+
|
111
|
+
info
|
112
|
+
end
|
113
|
+
|
114
|
+
def option_summary
|
115
|
+
options = []
|
116
|
+
optional_options = []
|
117
|
+
|
118
|
+
if self.options
|
119
|
+
self.options.collect{|option|
|
120
|
+
info = option_info(option)
|
121
|
+
if info[:default].nil?
|
122
|
+
options << info
|
123
|
+
else
|
124
|
+
optional_options << info
|
125
|
+
end
|
126
|
+
}
|
127
|
+
end
|
128
|
+
|
129
|
+
dependencies.select{|dep| Task === dep}.each do |task|
|
130
|
+
more_options, more_optional_options = task.option_summary
|
131
|
+
options.concat more_options
|
132
|
+
optional_options.concat more_optional_options
|
133
|
+
end
|
134
|
+
|
135
|
+
[options, optional_options]
|
136
|
+
end
|
137
|
+
|
138
|
+
def usage
|
139
|
+
usage = ""
|
140
|
+
usage << "Task: #{name}\n"
|
141
|
+
usage << "\nDescription: #{description.chomp}\n" if description
|
142
|
+
options, optional_options = option_summary
|
143
|
+
|
144
|
+
if options.any?
|
145
|
+
usage << "\nMandatory options:\n"
|
146
|
+
usage << "\tTask\tName\tType \tDescription\n"
|
147
|
+
usage << "\t----\t----\t---- \t-----------\n"
|
148
|
+
options.each do |option|
|
149
|
+
option_line = "\t[#{option[:source]}]\t#{option[:name]}"
|
150
|
+
option_line << "\t#{option[:type] ? option[:type] : "Unspec."}"
|
151
|
+
option_line << "\t#{option[:description]}" if option[:description]
|
152
|
+
usage << option_line << "\n"
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
if optional_options.any?
|
157
|
+
usage << "\nOptional options:"
|
158
|
+
usage << "Mandatory options:\n"
|
159
|
+
usage << "\tTask\tName\tDefault \tType \tDescription\n"
|
160
|
+
usage << "\t----\t----\t------- \t---- \t-----------\n"
|
161
|
+
optional_options.each do |option|
|
162
|
+
option_line = "\t[#{option[:source]}]\t#{option[:name]}\t#{option[:default]}"
|
163
|
+
option_line << "\t#{option[:type] ? option[:type] : "Unspec."}"
|
164
|
+
option_line << "\t#{option[:description]}" if option[:description]
|
165
|
+
usage << option_line << "\n"
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
usage
|
170
|
+
end
|
171
|
+
|
100
172
|
end
|
data/lib/rbbt/util/task/job.rb
CHANGED
@@ -82,8 +82,7 @@ class Task
|
|
82
82
|
end
|
83
83
|
|
84
84
|
def set_info(key, value)
|
85
|
-
Misc.lock(info_file, key, value) do |info_file, key, value|
|
86
|
-
i = self.info
|
85
|
+
Misc.lock(info_file, key, value) do |info_file, key, value| i = self.info
|
87
86
|
new_info = i.merge(key => value)
|
88
87
|
Open.write(info_file, new_info.to_yaml)
|
89
88
|
end
|
@@ -108,6 +107,23 @@ class Task
|
|
108
107
|
info[:messages] || []
|
109
108
|
end
|
110
109
|
|
110
|
+
def files(file = nil, data = nil)
|
111
|
+
return Dir.glob(File.join(path + '.files/*')).collect{|f| File.basename(f)} if file.nil?
|
112
|
+
|
113
|
+
filename = Resource::Path.path(File.join(path + '.files', file.to_s))
|
114
|
+
if data.nil?
|
115
|
+
filename
|
116
|
+
else
|
117
|
+
Open.write(filename, data)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def abort
|
122
|
+
if @pid
|
123
|
+
Process.kill("INT", @pid)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
111
127
|
def done?
|
112
128
|
[:done, :error, :aborted].include? info[:step]
|
113
129
|
end
|
@@ -244,7 +260,7 @@ class Task
|
|
244
260
|
File.open(path) do |f| f.read end
|
245
261
|
end
|
246
262
|
|
247
|
-
def load
|
263
|
+
def load(*args)
|
248
264
|
case task.persistence
|
249
265
|
when :float
|
250
266
|
Open.read(path).to_f
|
@@ -253,7 +269,7 @@ class Task
|
|
253
269
|
when :string
|
254
270
|
Open.read(path)
|
255
271
|
when :tsv
|
256
|
-
TSV.new(path)
|
272
|
+
TSV.new(path, *args)
|
257
273
|
when :marshal
|
258
274
|
Marshal.load(Open.read(path))
|
259
275
|
when :yaml
|
data/lib/rbbt/util/tc_hash.rb
CHANGED
@@ -93,7 +93,7 @@ class TCHash < TokyoCabinet::HDB
|
|
93
93
|
@serializer = Marshal
|
94
94
|
else
|
95
95
|
mod = Misc.string2const serializer_str
|
96
|
-
|
96
|
+
@serializer = mod
|
97
97
|
end
|
98
98
|
end
|
99
99
|
end
|
@@ -121,6 +121,7 @@ class TCHash < TokyoCabinet::HDB
|
|
121
121
|
@serializer = serializer
|
122
122
|
|
123
123
|
if write || ! File.exists?(@path_to_db)
|
124
|
+
self.setcache(100000) or raise "Error setting cache"
|
124
125
|
self.open(true)
|
125
126
|
else
|
126
127
|
self.open(false)
|