rbbt-util 5.34.4 → 5.34.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/association/database.rb +4 -0
- data/lib/rbbt/resource/rake.rb +9 -3
- data/lib/rbbt/resource/util.rb +8 -3
- data/lib/rbbt/resource.rb +1 -1
- data/lib/rbbt/tsv/accessor.rb +3 -0
- data/lib/rbbt/tsv/change_id.rb +1 -1
- data/lib/rbbt/tsv/manipulate.rb +13 -5
- data/lib/rbbt/tsv/parallel/traverse.rb +1 -1
- data/lib/rbbt/tsv/parser.rb +6 -4
- data/lib/rbbt/util/misc/development.rb +17 -12
- data/lib/rbbt/util/misc/inspect.rb +4 -2
- data/lib/rbbt/util/misc/omics.rb +1 -1
- data/lib/rbbt/util/open.rb +1 -1
- data/lib/rbbt/util/python.rb +1 -0
- data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +18 -1
- data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +5 -0
- data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +2 -1
- data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +9 -11
- data/lib/rbbt/workflow/step/save_load_inputs.rb +16 -5
- data/lib/rbbt/workflow/step/status.rb +1 -0
- data/lib/rbbt/workflow/step.rb +1 -1
- data/lib/rbbt/workflow/util/archive.rb +4 -0
- data/lib/rbbt/workflow/util/provenance.rb +1 -1
- data/share/Rlib/util.R +8 -2
- data/share/rbbt_commands/app/start +2 -0
- data/share/rbbt_commands/tsv/select +48 -0
- data/share/rbbt_commands/tsv/view +77 -0
- data/test/rbbt/test_resource.rb +14 -4
- metadata +4 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 57270d81ae9b5c8106f216ca39c6f3b7981e9fac5667fe5c3a89e4b42286bf8b
|
|
4
|
+
data.tar.gz: 841a63f9e5345fd8eb58db5878fb9556bb9d623fb2c78583a24688da8a518610
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3b4331f6fbbf25af0bd1bce2288fc5a39d06de0259ed74a2538ad785b2d48c2851321567c355dbbb612d8a478b8e40b8b16fe96ad3e497f61d28480795db88ee
|
|
7
|
+
data.tar.gz: 96621efdacb39998c672120f27347d92535fa8772381f39ae14e8e437f1ad7fe7c5176e3a0d6bcb54c6f89915aa5e928364af61e94f970d3fe6f2ba27d938408
|
|
@@ -162,6 +162,10 @@ module Association
|
|
|
162
162
|
|
|
163
163
|
data ||= {}
|
|
164
164
|
tsv = nil
|
|
165
|
+
if data.respond_to?(:close) && data.respond_to?(:write)
|
|
166
|
+
data.close
|
|
167
|
+
data.write
|
|
168
|
+
end
|
|
165
169
|
TmpFile.with_file do |tmpfile|
|
|
166
170
|
tmp_data = Persist.open_database(tmpfile, true, open_options[:type], "HDB")
|
|
167
171
|
|
data/lib/rbbt/resource/rake.rb
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
require 'rake'
|
|
2
2
|
module Rake
|
|
3
3
|
class TaskNotFound < StandardError; end
|
|
4
|
-
def self.run(rakefile, dir, task)
|
|
4
|
+
def self.run(rakefile, dir, task, &block)
|
|
5
5
|
old_pwd = FileUtils.pwd
|
|
6
6
|
|
|
7
7
|
Rake::FileTask.module_eval do
|
|
@@ -32,9 +32,15 @@ module Rake
|
|
|
32
32
|
t = nil
|
|
33
33
|
pid = Process.fork{
|
|
34
34
|
if block_given?
|
|
35
|
-
|
|
35
|
+
TOPLEVEL_BINDING.receiver.instance_exec &block
|
|
36
36
|
else
|
|
37
|
-
|
|
37
|
+
if Misc.is_filename? rakefile
|
|
38
|
+
load rakefile
|
|
39
|
+
else
|
|
40
|
+
TmpFile.with_file(rakefile) do |tmpfile|
|
|
41
|
+
load tmpfile
|
|
42
|
+
end
|
|
43
|
+
end
|
|
38
44
|
end
|
|
39
45
|
|
|
40
46
|
raise TaskNotFound if Rake::Task[task].nil?
|
data/lib/rbbt/resource/util.rb
CHANGED
|
@@ -123,8 +123,8 @@ module Resource
|
|
|
123
123
|
|
|
124
124
|
|
|
125
125
|
def rake_for(path)
|
|
126
|
-
@rake_dirs.
|
|
127
|
-
|
|
126
|
+
@rake_dirs.select{|dir, content|
|
|
127
|
+
Misc.common_path(dir, path)
|
|
128
128
|
}.sort_by{|dir, content|
|
|
129
129
|
dir.length
|
|
130
130
|
}.last
|
|
@@ -137,12 +137,17 @@ module Resource
|
|
|
137
137
|
def run_rake(path, rakefile, rake_dir)
|
|
138
138
|
task = Misc.path_relative_to rake_dir, path
|
|
139
139
|
rakefile = rakefile.produce if rakefile.respond_to? :produce
|
|
140
|
+
rakefile = rakefile.find if rakefile.respond_to? :find
|
|
140
141
|
|
|
141
142
|
rake_dir = rake_dir.find(:user) if rake_dir.respond_to? :find
|
|
142
143
|
|
|
143
144
|
begin
|
|
144
145
|
require 'rbbt/resource/rake'
|
|
145
|
-
|
|
146
|
+
if Proc === rakefile
|
|
147
|
+
Rake.run(nil, rake_dir, task, &rakefile)
|
|
148
|
+
else
|
|
149
|
+
Rake.run(rakefile, rake_dir, task)
|
|
150
|
+
end
|
|
146
151
|
rescue Rake::TaskNotFound
|
|
147
152
|
raise $! if rake_dir.nil? or rake_dir.empty? or rake_dir == "/" or rake_dir == "./"
|
|
148
153
|
task = File.join(File.basename(rake_dir), task)
|
data/lib/rbbt/resource.rb
CHANGED
data/lib/rbbt/tsv/accessor.rb
CHANGED
data/lib/rbbt/tsv/change_id.rb
CHANGED
data/lib/rbbt/tsv/manipulate.rb
CHANGED
|
@@ -527,13 +527,21 @@ module TSV
|
|
|
527
527
|
self.unnamed = old_unnamed
|
|
528
528
|
|
|
529
529
|
when String === method
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
530
|
+
if method =~ /^([<>]=?)(.*)/
|
|
531
|
+
with_unnamed do
|
|
532
|
+
through :key, key do |key, values|
|
|
533
|
+
value = Array === values ? values.flatten.first : values
|
|
534
|
+
new[key] = self[key] if value.to_f.send($1, $2.to_f)
|
|
535
|
+
end
|
|
536
|
+
end
|
|
537
|
+
else
|
|
538
|
+
with_unnamed do
|
|
539
|
+
through :key, key do |key, values|
|
|
540
|
+
values = [values] if type == :single
|
|
541
|
+
new[key] = self[key] if invert ^ (values.flatten.select{|v| v == method}.length > 0)
|
|
542
|
+
end
|
|
534
543
|
end
|
|
535
544
|
end
|
|
536
|
-
|
|
537
545
|
when Numeric === method
|
|
538
546
|
with_unnamed do
|
|
539
547
|
through :key, key do |key, values|
|
|
@@ -389,7 +389,7 @@ module TSV
|
|
|
389
389
|
when Set
|
|
390
390
|
traverse_array(obj.to_a, options, &block)
|
|
391
391
|
when String
|
|
392
|
-
if Open.remote?(obj)
|
|
392
|
+
if Open.remote?(obj) || Open.ssh?(obj) || Misc.is_filename?(obj)
|
|
393
393
|
Open.open(obj) do |s|
|
|
394
394
|
traverse_obj(s, options, &block)
|
|
395
395
|
end
|
data/lib/rbbt/tsv/parser.rb
CHANGED
|
@@ -8,7 +8,9 @@ module TSV
|
|
|
8
8
|
|
|
9
9
|
def all_fields
|
|
10
10
|
all = [key_field] + fields
|
|
11
|
-
|
|
11
|
+
# ToDo: What was this for?
|
|
12
|
+
#NamedArray.setup all, all
|
|
13
|
+
all
|
|
12
14
|
end
|
|
13
15
|
|
|
14
16
|
def parse_header(stream)
|
|
@@ -550,12 +552,12 @@ module TSV
|
|
|
550
552
|
line = self.rescue_first_line
|
|
551
553
|
line = stream.gets if line.nil?
|
|
552
554
|
|
|
553
|
-
if @tsv_grep
|
|
555
|
+
if @tsv_grep || grep
|
|
554
556
|
|
|
555
|
-
stream = Open.grep(stream, @tsv_grep, invert_grep, fixed_grep)
|
|
557
|
+
stream = Open.grep(stream, @tsv_grep || grep, invert_grep, fixed_grep)
|
|
556
558
|
stream.no_fail = true
|
|
557
559
|
begin
|
|
558
|
-
match = Open.grep(StringIO.new(line), @tsv_grep, invert_grep, fixed_grep).read
|
|
560
|
+
match = Open.grep(StringIO.new(line), @tsv_grep || grep, invert_grep, fixed_grep).read
|
|
559
561
|
line = stream.gets if match.empty?
|
|
560
562
|
rescue Exception
|
|
561
563
|
Log.exception $!
|
|
@@ -135,20 +135,25 @@ def self.add_libdir(dir=nil)
|
|
|
135
135
|
end
|
|
136
136
|
|
|
137
137
|
def self.insist(times = 4, sleep = nil, msg = nil)
|
|
138
|
-
|
|
139
|
-
sleep_array = times
|
|
140
|
-
times = sleep_array.length
|
|
141
|
-
sleep = sleep_array.shift
|
|
142
|
-
end
|
|
143
|
-
try = 0
|
|
144
|
-
|
|
145
|
-
if sleep.nil?
|
|
146
|
-
sleep_array = ([0] + [0.001, 0.01, 0.1, 0.5] * (times / 3)).sort[0..times-1]
|
|
147
|
-
sleep = sleep_array.shift
|
|
148
|
-
end
|
|
138
|
+
sleep_array = nil
|
|
149
139
|
|
|
140
|
+
try = 0
|
|
150
141
|
begin
|
|
151
|
-
|
|
142
|
+
begin
|
|
143
|
+
yield
|
|
144
|
+
rescue Exception
|
|
145
|
+
if Array === times
|
|
146
|
+
sleep_array = times
|
|
147
|
+
times = sleep_array.length
|
|
148
|
+
sleep = sleep_array.shift
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
if sleep.nil?
|
|
152
|
+
sleep_array = ([0] + [0.001, 0.01, 0.1, 0.5] * (times / 3)).sort[0..times-1]
|
|
153
|
+
sleep = sleep_array.shift
|
|
154
|
+
end
|
|
155
|
+
raise $!
|
|
156
|
+
end
|
|
152
157
|
rescue TryAgain
|
|
153
158
|
sleep sleep
|
|
154
159
|
retry
|
|
@@ -72,7 +72,9 @@ module Misc
|
|
|
72
72
|
when File
|
|
73
73
|
"<File:" + obj.path + ">"
|
|
74
74
|
when NamedArray
|
|
75
|
-
|
|
75
|
+
fields = obj.fields
|
|
76
|
+
fields = fields.collect if NamedArray === fields
|
|
77
|
+
"[<NamedArray: fields=#{fingerprint fields} -- values=#{fingerprint obj[0..-1]}]"
|
|
76
78
|
when Array
|
|
77
79
|
if (length = obj.length) > 10
|
|
78
80
|
"[#{length}--" << (obj.values_at(0,1, length / 2, -2, -1).collect{|e| fingerprint(e)} * ",") << "]"
|
|
@@ -299,7 +301,7 @@ module Misc
|
|
|
299
301
|
when nil
|
|
300
302
|
'nil'
|
|
301
303
|
when Numeric
|
|
302
|
-
obj.
|
|
304
|
+
Float === obj && obj % 1 == 0 ? obj.to_i.to_s : obj.to_s
|
|
303
305
|
when Symbol
|
|
304
306
|
obj.to_s
|
|
305
307
|
when TrueClass
|
data/lib/rbbt/util/misc/omics.rb
CHANGED
data/lib/rbbt/util/open.rb
CHANGED
data/lib/rbbt/util/python.rb
CHANGED
|
@@ -18,7 +18,24 @@ class RemoteWorkflow
|
|
|
18
18
|
end
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
+
def self.escape_url(url)
|
|
22
|
+
base, _sep, query = url.partition("?")
|
|
23
|
+
protocol, path = base.split("://")
|
|
24
|
+
path = protocol if path.nil?
|
|
25
|
+
path = path.split("/").collect{|p| CGI.escape(p) }* "/"
|
|
26
|
+
base = protocol ? [protocol, path] * "://" : path
|
|
27
|
+
|
|
28
|
+
if query && ! query.empty?
|
|
29
|
+
query = query.split("&").collect{|e| e.split("=").collect{|pe| CGI.escape(pe) } * "=" } * "&"
|
|
30
|
+
[base, query] * "?"
|
|
31
|
+
else
|
|
32
|
+
base
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
21
36
|
def self.encode(url)
|
|
37
|
+
# ToDo: Check this
|
|
38
|
+
return escape_url(url)
|
|
22
39
|
begin
|
|
23
40
|
URI::DEFAULT_PARSER.escape(url)
|
|
24
41
|
rescue
|
|
@@ -135,7 +152,7 @@ class RemoteWorkflow
|
|
|
135
152
|
|
|
136
153
|
def self.execute_job(base_url, task, task_params, cache_type)
|
|
137
154
|
RemoteWorkflow.capture_exception do
|
|
138
|
-
task_url =
|
|
155
|
+
task_url = self.escape_url(File.join(base_url, task.to_s))
|
|
139
156
|
|
|
140
157
|
sout, sin = Misc.pipe
|
|
141
158
|
|
|
@@ -148,6 +148,11 @@ job.clean
|
|
|
148
148
|
def self.upload_inputs(server, inputs, input_types, input_id)
|
|
149
149
|
TmpFile.with_file do |dir|
|
|
150
150
|
if Step.save_inputs(inputs, input_types, dir)
|
|
151
|
+
Dir.glob(File.join(dir, "*.as_step")).each do |file|
|
|
152
|
+
path = Open.read(file).strip
|
|
153
|
+
new = Step.migrate(path, :user, :target => server)
|
|
154
|
+
Open.write(file, new)
|
|
155
|
+
end
|
|
151
156
|
CMD.cmd("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/; scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
|
|
152
157
|
end
|
|
153
158
|
end
|
|
@@ -132,7 +132,7 @@ class RemoteStep
|
|
|
132
132
|
def _run_job(cache_type = :asynchronous)
|
|
133
133
|
get_streams
|
|
134
134
|
|
|
135
|
-
task_url =
|
|
135
|
+
task_url = RemoteWorkflow::REST.escape_url(File.join(base_url, task.to_s))
|
|
136
136
|
@adaptor.__prepare_inputs_for_restclient(inputs)
|
|
137
137
|
task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
|
|
138
138
|
|
|
@@ -147,6 +147,7 @@ class RemoteStep
|
|
|
147
147
|
|
|
148
148
|
def produce(*args)
|
|
149
149
|
@started = true
|
|
150
|
+
init_job
|
|
150
151
|
_run_job
|
|
151
152
|
end
|
|
152
153
|
|
|
@@ -3,6 +3,7 @@ class RemoteStep
|
|
|
3
3
|
attr_accessor :override_dependencies
|
|
4
4
|
|
|
5
5
|
def init_job(cache_type = nil, other_params = {})
|
|
6
|
+
return self if @url
|
|
6
7
|
cache_type = :asynchronous if cache_type.nil? and not @is_exec
|
|
7
8
|
cache_type = :exec if cache_type.nil?
|
|
8
9
|
@last_info_time = nil
|
|
@@ -10,20 +11,17 @@ class RemoteStep
|
|
|
10
11
|
@server, @server_path = RemoteWorkflow::SSH.parse_url base_url
|
|
11
12
|
@input_id ||= "inputs-" << rand(100000).to_s
|
|
12
13
|
|
|
13
|
-
if override_dependencies
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
name, _sep, value = od.partition("=")
|
|
18
|
-
inputs[name] = value
|
|
19
|
-
end
|
|
14
|
+
if override_dependencies && override_dependencies.any?
|
|
15
|
+
override_dependencies.each do |od|
|
|
16
|
+
name, _sep, value = od.partition("=")
|
|
17
|
+
inputs[name] = value
|
|
20
18
|
end
|
|
21
|
-
|
|
22
|
-
RemoteWorkflow::SSH.upload_inputs(@server, inputs, @input_types, @input_id)
|
|
23
|
-
else
|
|
24
|
-
RemoteWorkflow::SSH.upload_inputs(@server, inputs, @input_types, @input_id)
|
|
25
19
|
end
|
|
26
20
|
|
|
21
|
+
inputs.select{|i| Step === i }.each{|i| i.produce }
|
|
22
|
+
|
|
23
|
+
RemoteWorkflow::SSH.upload_inputs(@server, inputs, @input_types, @input_id)
|
|
24
|
+
|
|
27
25
|
@name ||= Persist.memory("RemoteSteps", :workflow => self, :task => task, :jobname => @name, :inputs => inputs, :cache_type => cache_type) do
|
|
28
26
|
Misc.insist do
|
|
29
27
|
input_types = {}
|
|
@@ -51,8 +51,8 @@ module Workflow
|
|
|
51
51
|
when :path
|
|
52
52
|
inputs[input.to_sym] = Open.read(file).strip.split("\n").first
|
|
53
53
|
when :io
|
|
54
|
-
inputs[input.to_sym] = Open.open(Open.realpath(file))
|
|
55
|
-
when :
|
|
54
|
+
inputs[input.to_sym] = Open.open(Open.realpath(file))
|
|
55
|
+
when :io_array
|
|
56
56
|
inputs[input.to_sym] = Open.realpath(file).split("\n").collect{|f| Open.open(f)}
|
|
57
57
|
when :step_array
|
|
58
58
|
steps = Open.read(file).strip.split("\n").collect{|path| Workflow.load_step(path) }
|
|
@@ -97,6 +97,10 @@ module Workflow
|
|
|
97
97
|
inputs[input.to_sym] = TSV.open(file)
|
|
98
98
|
when :boolean
|
|
99
99
|
inputs[input.to_sym] = (file.read.strip == 'true')
|
|
100
|
+
when :integer
|
|
101
|
+
inputs[input.to_sym] = file.read.to_i
|
|
102
|
+
when :float
|
|
103
|
+
inputs[input.to_sym] = file.read.to_f
|
|
100
104
|
else
|
|
101
105
|
Log.debug "Loading #{ input } from #{file}"
|
|
102
106
|
inputs[input.to_sym] = file.read.strip
|
|
@@ -172,10 +176,17 @@ class Step
|
|
|
172
176
|
end
|
|
173
177
|
|
|
174
178
|
Log.debug "Saving job input #{name} (#{type}) into #{path}"
|
|
175
|
-
|
|
179
|
+
|
|
180
|
+
if IO === value && value.respond_to?(:filename) && value.filename
|
|
181
|
+
Open.write(path, value.filename)
|
|
182
|
+
elsif IO === value
|
|
183
|
+
Open.write(path, value)
|
|
184
|
+
else
|
|
185
|
+
Open.write(path, value.to_s)
|
|
186
|
+
end
|
|
176
187
|
end
|
|
177
188
|
|
|
178
|
-
def self.save_inputs(inputs, input_types,
|
|
189
|
+
def self.save_inputs(inputs, input_types, dir)
|
|
179
190
|
inputs.each do |name,value|
|
|
180
191
|
type = input_types[name]
|
|
181
192
|
type = type.to_s if type
|
|
@@ -220,7 +231,7 @@ class Step
|
|
|
220
231
|
input_types = IndiferentHash.setup(input_types.merge(:override_dependencies => :array))
|
|
221
232
|
end
|
|
222
233
|
|
|
223
|
-
save_inputs(inputs, input_types,
|
|
234
|
+
save_inputs(inputs, input_types, dir)
|
|
224
235
|
|
|
225
236
|
inputs.keys
|
|
226
237
|
end
|
|
@@ -75,6 +75,7 @@ class Step
|
|
|
75
75
|
|
|
76
76
|
def dirty?
|
|
77
77
|
return true if Open.exists?(pid_file) && ! ( Open.exists?(info_file) || done? )
|
|
78
|
+
return true if done? && ! (status == :done || status == :noinfo)
|
|
78
79
|
return false unless done? || status == :done
|
|
79
80
|
return false unless ENV["RBBT_UPDATE"] == "true"
|
|
80
81
|
|
data/lib/rbbt/workflow/step.rb
CHANGED
|
@@ -184,9 +184,13 @@ puts files * "\n"
|
|
|
184
184
|
|
|
185
185
|
target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
|
|
186
186
|
|
|
187
|
+
target_path = File.join(target, *path.split("/")[-3..-1])
|
|
188
|
+
|
|
187
189
|
subpath_files.each do |subpath, files|
|
|
188
190
|
Rbbt.migrate_files([subpath], target, options.merge(:files => files))
|
|
189
191
|
end
|
|
192
|
+
|
|
193
|
+
target_path
|
|
190
194
|
end
|
|
191
195
|
|
|
192
196
|
def self.purge(path, recursive = false, skip_overriden = true)
|
data/share/Rlib/util.R
CHANGED
|
@@ -232,20 +232,26 @@ rbbt.tsv2tibble <- function(data){
|
|
|
232
232
|
as_tibble(data, rownames=attr(data, 'key.field'))
|
|
233
233
|
}
|
|
234
234
|
|
|
235
|
-
rbbt.tsv.write <- function(filename, data, key.field = NULL, extra_headers = NULL, eol="\n", ...){
|
|
235
|
+
rbbt.tsv.write <- function(filename, data, key.field = NULL, extra_headers = NULL, eol="\n", type = 'list', names = NULL, ...){
|
|
236
236
|
|
|
237
237
|
if (is.null(key.field)){ key.field = attributes(data)$key.field;}
|
|
238
238
|
if (is.null(key.field)){ key.field = "ID";}
|
|
239
239
|
|
|
240
240
|
f = file(filename, 'wb');
|
|
241
241
|
|
|
242
|
+
if (is.null(extra_headers)){
|
|
243
|
+
extra_headers = paste(":type",type,sep="=")
|
|
244
|
+
}
|
|
245
|
+
|
|
242
246
|
if (!is.null(extra_headers)){
|
|
243
247
|
extra_headers = paste("#: ", extra_headers, "\n", sep="");
|
|
244
248
|
cat(extra_headers, file=f);
|
|
245
249
|
}
|
|
246
250
|
|
|
247
251
|
header = paste("#", key.field, sep="");
|
|
248
|
-
|
|
252
|
+
if (is.null(names)){ names = colnames(data)}
|
|
253
|
+
if (is.null(names)){ names = names(data)}
|
|
254
|
+
for (name in names){ header = paste(header, name, sep="\t");}
|
|
249
255
|
header = paste(header, "\n", sep="");
|
|
250
256
|
cat(header, file=f);
|
|
251
257
|
|
|
@@ -96,6 +96,8 @@ Misc.in_dir(app_dir) do
|
|
|
96
96
|
system ENV, "env RBBT_LOG=0 passenger start -R '#{config_ru_file}' -p #{options[:Port] || "2887"}"
|
|
97
97
|
when 'puma_alt'
|
|
98
98
|
system ENV, "puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 3 -t 8:32 --preload"
|
|
99
|
+
when 'puma_production'
|
|
100
|
+
CMD.cmd_log("env RBBT_LOG=#{Log.severity.to_s} puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 20 -t 10:160 --preload")
|
|
99
101
|
else
|
|
100
102
|
options[:config] = config_ru_file
|
|
101
103
|
options[:threads] = "8:8"
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require 'rbbt-util'
|
|
4
|
+
require 'rbbt/util/simpleopt'
|
|
5
|
+
|
|
6
|
+
$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
|
|
7
|
+
|
|
8
|
+
options = SOPT.setup <<EOF
|
|
9
|
+
Select entries in a tsv based on some criteria
|
|
10
|
+
|
|
11
|
+
$ rbbt tsv subset [options] file.tsv <key> <criteria>
|
|
12
|
+
|
|
13
|
+
Subsets entries from a TSV file from a given list. Works with Tokyocabinet HDB and BDB as well.
|
|
14
|
+
|
|
15
|
+
-tch--tokyocabinet File is a TC HDB
|
|
16
|
+
-tcb--tokyocabinet_bd File is a TC BDB
|
|
17
|
+
-hh--header_hash* Change the character used to mark the header line (defaults to #)
|
|
18
|
+
-s--subset* Subset of keys (Comma-separated or file)
|
|
19
|
+
-m--merge Merge TSV rows
|
|
20
|
+
-h--help Help
|
|
21
|
+
EOF
|
|
22
|
+
|
|
23
|
+
SOPT.usage if options[:help]
|
|
24
|
+
|
|
25
|
+
file, key, criteria = ARGV
|
|
26
|
+
|
|
27
|
+
key, criteria, file = [file, key, nil] if criteria.nil?
|
|
28
|
+
|
|
29
|
+
file = STDIN if file == '-' || file.nil?
|
|
30
|
+
|
|
31
|
+
raise ParameterException, "Please specify the tsv file as argument" if file.nil?
|
|
32
|
+
|
|
33
|
+
options[:fields] = options[:fields].split(/,\|/) if options[:fields]
|
|
34
|
+
options[:header_hash] = options["header_hash"]
|
|
35
|
+
|
|
36
|
+
case
|
|
37
|
+
when options[:tokyocabinet]
|
|
38
|
+
tsv = Persist.open_tokyocabinet(file, false)
|
|
39
|
+
puts tsv.summary
|
|
40
|
+
when options[:tokyocabinet_bd]
|
|
41
|
+
tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
|
|
42
|
+
puts tsv.summary
|
|
43
|
+
else
|
|
44
|
+
tsv = TSV.open(file, options)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
criteria = criteria.to_regexp.to_regexp if criteria =~ /\/.*\/.*/
|
|
48
|
+
puts tsv.select(key => criteria)
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require 'rbbt-util'
|
|
4
|
+
require 'rbbt/util/simpleopt'
|
|
5
|
+
|
|
6
|
+
$0 = "rbbt #{$previous_commands*" "} #{ File.basename(__FILE__) }" if $previous_commands
|
|
7
|
+
|
|
8
|
+
options = SOPT.setup <<EOF
|
|
9
|
+
|
|
10
|
+
View TSV nicer
|
|
11
|
+
|
|
12
|
+
$ #{$0} [options] <filename.tsv|->
|
|
13
|
+
|
|
14
|
+
Use - to read from STDIN
|
|
15
|
+
|
|
16
|
+
-h--help Print this help
|
|
17
|
+
|
|
18
|
+
EOF
|
|
19
|
+
if options[:help]
|
|
20
|
+
if defined? rbbt_usage
|
|
21
|
+
rbbt_usage
|
|
22
|
+
else
|
|
23
|
+
puts SOPT.doc
|
|
24
|
+
end
|
|
25
|
+
exit 0
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
file = ARGV.shift
|
|
30
|
+
|
|
31
|
+
file = STDIN if file == '-' || file.nil?
|
|
32
|
+
|
|
33
|
+
case
|
|
34
|
+
when options[:tokyocabinet]
|
|
35
|
+
tsv = Persist.open_tokyocabinet(file, false)
|
|
36
|
+
when options[:tokyocabinet_bd]
|
|
37
|
+
tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
|
|
38
|
+
else
|
|
39
|
+
if String === file
|
|
40
|
+
file = file.dup
|
|
41
|
+
Path.setup(File.expand_path(file))
|
|
42
|
+
end
|
|
43
|
+
tsv = file
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
tsv = TSV.open(tsv) if IO === tsv
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
begin
|
|
50
|
+
require 'rbbt/workflow'
|
|
51
|
+
Workflow.require_workflow "Genomics"
|
|
52
|
+
tsv = Genomics.job(:names, nil, :tsv => tsv).run
|
|
53
|
+
rescue
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
puts [Log.highlight(Log.color(:magenta, tsv.key_field)), Log.color(:yellow, tsv.fields * "\t")] * "\t"
|
|
57
|
+
TSV.traverse tsv do |k,values,fields|
|
|
58
|
+
fvalues = values.zip(fields).collect do |v,field|
|
|
59
|
+
if String === v
|
|
60
|
+
v = v.to_f.round(2) if v =~ /^-?\d+\.\d+$/
|
|
61
|
+
v = v.to_i if v =~ /^-?\d+$/
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
if Numeric === v
|
|
65
|
+
if field =~ /p.?value/i
|
|
66
|
+
v = v.round(4)
|
|
67
|
+
v < 0.05 ? Log.color(:cyan, v.to_s) : v
|
|
68
|
+
else
|
|
69
|
+
v = v.round(2)
|
|
70
|
+
v < 0 ? Log.color(:red, v.to_s) : Log.color(:green, v.to_s)
|
|
71
|
+
end
|
|
72
|
+
else
|
|
73
|
+
v
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
puts [Log.color(:magenta, k), fvalues] * "\t"
|
|
77
|
+
end
|
data/test/rbbt/test_resource.rb
CHANGED
|
@@ -17,9 +17,18 @@ module TestResource
|
|
|
17
17
|
file 'foo' do |t|
|
|
18
18
|
Open.write(t.name, "TEST")
|
|
19
19
|
end
|
|
20
|
+
|
|
21
|
+
rule /.*/ do |t|
|
|
22
|
+
Open.write(t.name, "bar")
|
|
23
|
+
end
|
|
20
24
|
EOF
|
|
21
25
|
|
|
22
|
-
claim tmp.test.work.footest, :rake, tmp.test.rakefiles.foo
|
|
26
|
+
claim tmp.test.work.footest, :rake, TestResource.tmp.test.rakefiles.foo
|
|
27
|
+
|
|
28
|
+
claim tmp.test.work.file_proc, :file_proc do |file,filename|
|
|
29
|
+
Open.write(filename, file)
|
|
30
|
+
nil
|
|
31
|
+
end
|
|
23
32
|
end
|
|
24
33
|
|
|
25
34
|
class TestTSV < Test::Unit::TestCase
|
|
@@ -33,11 +42,12 @@ class TestTSV < Test::Unit::TestCase
|
|
|
33
42
|
assert TSV === TestResource.tmp.test.test_tsv.tsv
|
|
34
43
|
end
|
|
35
44
|
|
|
36
|
-
def
|
|
37
|
-
TestResource.tmp.test.work.footest.foo.read
|
|
38
|
-
|
|
45
|
+
def test_rake
|
|
46
|
+
assert_equal TestResource.tmp.test.work.footest.foo.read, "TEST"
|
|
47
|
+
assert_equal TestResource.tmp.test.work.footest.bar.read, "bar"
|
|
39
48
|
end
|
|
40
49
|
|
|
50
|
+
|
|
41
51
|
def test_proc
|
|
42
52
|
assert TestResource.tmp.test.proc.read == "PROC TEST"
|
|
43
53
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: rbbt-util
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 5.34.
|
|
4
|
+
version: 5.34.8
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Miguel Vazquez
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2022-07-
|
|
11
|
+
date: 2022-07-27 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rake
|
|
@@ -452,6 +452,7 @@ files:
|
|
|
452
452
|
- share/rbbt_commands/tsv/query
|
|
453
453
|
- share/rbbt_commands/tsv/read
|
|
454
454
|
- share/rbbt_commands/tsv/read_excel
|
|
455
|
+
- share/rbbt_commands/tsv/select
|
|
455
456
|
- share/rbbt_commands/tsv/slice
|
|
456
457
|
- share/rbbt_commands/tsv/sort
|
|
457
458
|
- share/rbbt_commands/tsv/subset
|
|
@@ -459,6 +460,7 @@ files:
|
|
|
459
460
|
- share/rbbt_commands/tsv/uncollapse
|
|
460
461
|
- share/rbbt_commands/tsv/unzip
|
|
461
462
|
- share/rbbt_commands/tsv/values
|
|
463
|
+
- share/rbbt_commands/tsv/view
|
|
462
464
|
- share/rbbt_commands/tsv/write_excel
|
|
463
465
|
- share/rbbt_commands/tsv/zip
|
|
464
466
|
- share/rbbt_commands/watch
|