rbbt-util 5.34.3 → 5.34.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/association/database.rb +4 -0
- data/lib/rbbt/resource/rake.rb +9 -3
- data/lib/rbbt/resource/util.rb +8 -3
- data/lib/rbbt/resource.rb +5 -2
- data/lib/rbbt/tsv/accessor.rb +3 -0
- data/lib/rbbt/tsv/change_id.rb +1 -1
- data/lib/rbbt/tsv/parallel/traverse.rb +1 -1
- data/lib/rbbt/tsv/parser.rb +6 -4
- data/lib/rbbt/util/misc/development.rb +17 -12
- data/lib/rbbt/util/misc/inspect.rb +4 -2
- data/lib/rbbt/util/misc/omics.rb +1 -1
- data/lib/rbbt/util/misc/pipes.rb +1 -1
- data/lib/rbbt/util/open.rb +1 -1
- data/lib/rbbt/util/python.rb +9 -1
- data/lib/rbbt/workflow/definition.rb +7 -2
- data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +18 -1
- data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +5 -0
- data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +2 -1
- data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +9 -11
- data/lib/rbbt/workflow/step/save_load_inputs.rb +16 -5
- data/lib/rbbt/workflow/step/status.rb +1 -0
- data/lib/rbbt/workflow/step.rb +1 -1
- data/lib/rbbt/workflow/util/archive.rb +4 -0
- data/lib/rbbt/workflow/util/provenance.rb +1 -1
- data/share/Rlib/util.R +8 -2
- data/share/rbbt_commands/app/start +2 -0
- data/share/rbbt_commands/tsv/view +77 -0
- data/test/rbbt/test_resource.rb +14 -4
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ec5860171e51e2e53a8383ae8ea1156b79c7f75194d384c5aa2c43efd759e129
|
4
|
+
data.tar.gz: a6ab8a782b3169eae1e475a2029e8a4f0ac2d12f375475ec70de9d97f513f21a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2fb6a8ac5abee267891f6bb0736b659f202019de75fa0ecef80e955838a54013eaeabb495c44daff45196a8706821b706bfae75b095ce3ca8b82e8351436d4ef
|
7
|
+
data.tar.gz: ab24e4ab588e75f94adfcd9a02e178511bd4471edadef18c6c49537974353b8b4d83c61438d8fe638acbc26101556637a92d5da325ebc135736da44558971334
|
@@ -162,6 +162,10 @@ module Association
|
|
162
162
|
|
163
163
|
data ||= {}
|
164
164
|
tsv = nil
|
165
|
+
if data.respond_to?(:close) && data.respond_to?(:write)
|
166
|
+
data.close
|
167
|
+
data.write
|
168
|
+
end
|
165
169
|
TmpFile.with_file do |tmpfile|
|
166
170
|
tmp_data = Persist.open_database(tmpfile, true, open_options[:type], "HDB")
|
167
171
|
|
data/lib/rbbt/resource/rake.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'rake'
|
2
2
|
module Rake
|
3
3
|
class TaskNotFound < StandardError; end
|
4
|
-
def self.run(rakefile, dir, task)
|
4
|
+
def self.run(rakefile, dir, task, &block)
|
5
5
|
old_pwd = FileUtils.pwd
|
6
6
|
|
7
7
|
Rake::FileTask.module_eval do
|
@@ -32,9 +32,15 @@ module Rake
|
|
32
32
|
t = nil
|
33
33
|
pid = Process.fork{
|
34
34
|
if block_given?
|
35
|
-
|
35
|
+
TOPLEVEL_BINDING.receiver.instance_exec &block
|
36
36
|
else
|
37
|
-
|
37
|
+
if Misc.is_filename? rakefile
|
38
|
+
load rakefile
|
39
|
+
else
|
40
|
+
TmpFile.with_file(rakefile) do |tmpfile|
|
41
|
+
load tmpfile
|
42
|
+
end
|
43
|
+
end
|
38
44
|
end
|
39
45
|
|
40
46
|
raise TaskNotFound if Rake::Task[task].nil?
|
data/lib/rbbt/resource/util.rb
CHANGED
@@ -123,8 +123,8 @@ module Resource
|
|
123
123
|
|
124
124
|
|
125
125
|
def rake_for(path)
|
126
|
-
@rake_dirs.
|
127
|
-
|
126
|
+
@rake_dirs.select{|dir, content|
|
127
|
+
Misc.common_path(dir, path)
|
128
128
|
}.sort_by{|dir, content|
|
129
129
|
dir.length
|
130
130
|
}.last
|
@@ -137,12 +137,17 @@ module Resource
|
|
137
137
|
def run_rake(path, rakefile, rake_dir)
|
138
138
|
task = Misc.path_relative_to rake_dir, path
|
139
139
|
rakefile = rakefile.produce if rakefile.respond_to? :produce
|
140
|
+
rakefile = rakefile.find if rakefile.respond_to? :find
|
140
141
|
|
141
142
|
rake_dir = rake_dir.find(:user) if rake_dir.respond_to? :find
|
142
143
|
|
143
144
|
begin
|
144
145
|
require 'rbbt/resource/rake'
|
145
|
-
|
146
|
+
if Proc === rakefile
|
147
|
+
Rake.run(nil, rake_dir, task, &rakefile)
|
148
|
+
else
|
149
|
+
Rake.run(rakefile, rake_dir, task)
|
150
|
+
end
|
146
151
|
rescue Rake::TaskNotFound
|
147
152
|
raise $! if rake_dir.nil? or rake_dir.empty? or rake_dir == "/" or rake_dir == "./"
|
148
153
|
task = File.join(File.basename(rake_dir), task)
|
data/lib/rbbt/resource.rb
CHANGED
@@ -59,7 +59,7 @@ module Resource
|
|
59
59
|
|
60
60
|
def claim(path, type, content = nil, &block)
|
61
61
|
if type == :rake
|
62
|
-
@rake_dirs[path] = content
|
62
|
+
@rake_dirs[path] = content || block
|
63
63
|
else
|
64
64
|
@resources[path] = [type, content || block]
|
65
65
|
|
@@ -346,9 +346,12 @@ url='#{url}'
|
|
346
346
|
locations = (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys)
|
347
347
|
locations -= [:current, "current"]
|
348
348
|
locations << :current
|
349
|
+
search_paths = IndiferentHash.setup(resource.search_paths)
|
349
350
|
locations.uniq.each do |name|
|
350
|
-
pattern =
|
351
|
+
pattern = search_paths[name]
|
352
|
+
pattern = resource.search_paths[pattern] while Symbol === pattern
|
351
353
|
next if pattern.nil?
|
354
|
+
|
352
355
|
pattern = pattern.sub('{PWD}', Dir.pwd)
|
353
356
|
if String === pattern and pattern.include?('{')
|
354
357
|
regexp = "^" + pattern.gsub(/{([^}]+)}/,'(?<\1>[^/]+)') + "(?:/(?<REST>.*))?/?$"
|
data/lib/rbbt/tsv/accessor.rb
CHANGED
data/lib/rbbt/tsv/change_id.rb
CHANGED
@@ -389,7 +389,7 @@ module TSV
|
|
389
389
|
when Set
|
390
390
|
traverse_array(obj.to_a, options, &block)
|
391
391
|
when String
|
392
|
-
if Open.remote?(obj)
|
392
|
+
if Open.remote?(obj) || Open.ssh?(obj) || Misc.is_filename?(obj)
|
393
393
|
Open.open(obj) do |s|
|
394
394
|
traverse_obj(s, options, &block)
|
395
395
|
end
|
data/lib/rbbt/tsv/parser.rb
CHANGED
@@ -8,7 +8,9 @@ module TSV
|
|
8
8
|
|
9
9
|
def all_fields
|
10
10
|
all = [key_field] + fields
|
11
|
-
|
11
|
+
# ToDo: What was this for?
|
12
|
+
#NamedArray.setup all, all
|
13
|
+
all
|
12
14
|
end
|
13
15
|
|
14
16
|
def parse_header(stream)
|
@@ -550,12 +552,12 @@ module TSV
|
|
550
552
|
line = self.rescue_first_line
|
551
553
|
line = stream.gets if line.nil?
|
552
554
|
|
553
|
-
if @tsv_grep
|
555
|
+
if @tsv_grep || grep
|
554
556
|
|
555
|
-
stream = Open.grep(stream, @tsv_grep, invert_grep, fixed_grep)
|
557
|
+
stream = Open.grep(stream, @tsv_grep || grep, invert_grep, fixed_grep)
|
556
558
|
stream.no_fail = true
|
557
559
|
begin
|
558
|
-
match = Open.grep(StringIO.new(line), @tsv_grep, invert_grep, fixed_grep).read
|
560
|
+
match = Open.grep(StringIO.new(line), @tsv_grep || grep, invert_grep, fixed_grep).read
|
559
561
|
line = stream.gets if match.empty?
|
560
562
|
rescue Exception
|
561
563
|
Log.exception $!
|
@@ -135,20 +135,25 @@ def self.add_libdir(dir=nil)
|
|
135
135
|
end
|
136
136
|
|
137
137
|
def self.insist(times = 4, sleep = nil, msg = nil)
|
138
|
-
|
139
|
-
sleep_array = times
|
140
|
-
times = sleep_array.length
|
141
|
-
sleep = sleep_array.shift
|
142
|
-
end
|
143
|
-
try = 0
|
144
|
-
|
145
|
-
if sleep.nil?
|
146
|
-
sleep_array = ([0] + [0.001, 0.01, 0.1, 0.5] * (times / 3)).sort[0..times-1]
|
147
|
-
sleep = sleep_array.shift
|
148
|
-
end
|
138
|
+
sleep_array = nil
|
149
139
|
|
140
|
+
try = 0
|
150
141
|
begin
|
151
|
-
|
142
|
+
begin
|
143
|
+
yield
|
144
|
+
rescue Exception
|
145
|
+
if Array === times
|
146
|
+
sleep_array = times
|
147
|
+
times = sleep_array.length
|
148
|
+
sleep = sleep_array.shift
|
149
|
+
end
|
150
|
+
|
151
|
+
if sleep.nil?
|
152
|
+
sleep_array = ([0] + [0.001, 0.01, 0.1, 0.5] * (times / 3)).sort[0..times-1]
|
153
|
+
sleep = sleep_array.shift
|
154
|
+
end
|
155
|
+
raise $!
|
156
|
+
end
|
152
157
|
rescue TryAgain
|
153
158
|
sleep sleep
|
154
159
|
retry
|
@@ -72,7 +72,9 @@ module Misc
|
|
72
72
|
when File
|
73
73
|
"<File:" + obj.path + ">"
|
74
74
|
when NamedArray
|
75
|
-
|
75
|
+
fields = obj.fields
|
76
|
+
fields = fields.collect if NamedArray === fields
|
77
|
+
"[<NamedArray: fields=#{fingerprint fields} -- values=#{fingerprint obj[0..-1]}]"
|
76
78
|
when Array
|
77
79
|
if (length = obj.length) > 10
|
78
80
|
"[#{length}--" << (obj.values_at(0,1, length / 2, -2, -1).collect{|e| fingerprint(e)} * ",") << "]"
|
@@ -299,7 +301,7 @@ module Misc
|
|
299
301
|
when nil
|
300
302
|
'nil'
|
301
303
|
when Numeric
|
302
|
-
obj.
|
304
|
+
Float === obj && obj % 1 == 0 ? obj.to_i.to_s : obj.to_s
|
303
305
|
when Symbol
|
304
306
|
obj.to_s
|
305
307
|
when TrueClass
|
data/lib/rbbt/util/misc/omics.rb
CHANGED
data/lib/rbbt/util/misc/pipes.rb
CHANGED
@@ -420,7 +420,7 @@ module Misc
|
|
420
420
|
end
|
421
421
|
|
422
422
|
Open.touch path if Open.exists? path
|
423
|
-
content.join if content.respond_to?
|
423
|
+
content.join if content.respond_to?(:join) and not Path === content and not (content.respond_to?(:joined?) && content.joined?)
|
424
424
|
|
425
425
|
Open.notify_write(path)
|
426
426
|
rescue Aborted
|
data/lib/rbbt/util/open.rb
CHANGED
data/lib/rbbt/util/python.rb
CHANGED
@@ -1,9 +1,15 @@
|
|
1
1
|
require 'rbbt-util'
|
2
2
|
require 'pycall/import'
|
3
|
+
require 'rbbt/util/python/util'
|
3
4
|
|
4
5
|
module RbbtPython
|
5
6
|
extend PyCall::Import
|
6
7
|
|
8
|
+
def self.script(text, options = {})
|
9
|
+
Log.debug "Running python script:\n#{text.dup}"
|
10
|
+
text = StringIO.new text unless IO === text
|
11
|
+
CMD.cmd_log(:python, options.merge(:in => text))
|
12
|
+
end
|
7
13
|
|
8
14
|
def self.add_path(path)
|
9
15
|
self.run 'sys' do
|
@@ -87,7 +93,9 @@ module RbbtPython
|
|
87
93
|
|
88
94
|
def self.run_log(mod = nil, imports = nil, severity = 0, severity_err = nil, &block)
|
89
95
|
if mod
|
90
|
-
if
|
96
|
+
if imports == "*" || imports == ["*"]
|
97
|
+
pyfrom mod
|
98
|
+
elsif Array === imports
|
91
99
|
pyfrom mod, :import => imports
|
92
100
|
elsif Hash === imports
|
93
101
|
pyimport mod, imports
|
@@ -22,8 +22,13 @@ module Workflow
|
|
22
22
|
:extension => nil)
|
23
23
|
|
24
24
|
|
25
|
-
def helper(name, &block)
|
26
|
-
|
25
|
+
def helper(name, *args, &block)
|
26
|
+
if block_given?
|
27
|
+
helpers[name] = block
|
28
|
+
else
|
29
|
+
raise RbbtException, "helper #{name} unkown in #{self} workflow" unless helpers[name]
|
30
|
+
helpers[name].call(*args)
|
31
|
+
end
|
27
32
|
end
|
28
33
|
|
29
34
|
def desc(description)
|
@@ -18,7 +18,24 @@ class RemoteWorkflow
|
|
18
18
|
end
|
19
19
|
end
|
20
20
|
|
21
|
+
def self.escape_url(url)
|
22
|
+
base, _sep, query = url.partition("?")
|
23
|
+
protocol, path = base.split("://")
|
24
|
+
path = protocol if path.nil?
|
25
|
+
path = path.split("/").collect{|p| CGI.escape(p) }* "/"
|
26
|
+
base = protocol ? [protocol, path] * "://" : path
|
27
|
+
|
28
|
+
if query && ! query.empty?
|
29
|
+
query = query.split("&").collect{|e| e.split("=").collect{|pe| CGI.escape(pe) } * "=" } * "&"
|
30
|
+
[base, query] * "?"
|
31
|
+
else
|
32
|
+
base
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
21
36
|
def self.encode(url)
|
37
|
+
# ToDo: Check this
|
38
|
+
return escape_url(url)
|
22
39
|
begin
|
23
40
|
URI::DEFAULT_PARSER.escape(url)
|
24
41
|
rescue
|
@@ -135,7 +152,7 @@ class RemoteWorkflow
|
|
135
152
|
|
136
153
|
def self.execute_job(base_url, task, task_params, cache_type)
|
137
154
|
RemoteWorkflow.capture_exception do
|
138
|
-
task_url =
|
155
|
+
task_url = self.escape_url(File.join(base_url, task.to_s))
|
139
156
|
|
140
157
|
sout, sin = Misc.pipe
|
141
158
|
|
@@ -148,6 +148,11 @@ job.clean
|
|
148
148
|
def self.upload_inputs(server, inputs, input_types, input_id)
|
149
149
|
TmpFile.with_file do |dir|
|
150
150
|
if Step.save_inputs(inputs, input_types, dir)
|
151
|
+
Dir.glob(File.join(dir, "*.as_step")).each do |file|
|
152
|
+
path = Open.read(file).strip
|
153
|
+
new = Step.migrate(path, :user, :target => server)
|
154
|
+
Open.write(file, new)
|
155
|
+
end
|
151
156
|
CMD.cmd("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/; scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
|
152
157
|
end
|
153
158
|
end
|
@@ -132,7 +132,7 @@ class RemoteStep
|
|
132
132
|
def _run_job(cache_type = :asynchronous)
|
133
133
|
get_streams
|
134
134
|
|
135
|
-
task_url =
|
135
|
+
task_url = RemoteWorkflow::REST.escape_url(File.join(base_url, task.to_s))
|
136
136
|
@adaptor.__prepare_inputs_for_restclient(inputs)
|
137
137
|
task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
|
138
138
|
|
@@ -147,6 +147,7 @@ class RemoteStep
|
|
147
147
|
|
148
148
|
def produce(*args)
|
149
149
|
@started = true
|
150
|
+
init_job
|
150
151
|
_run_job
|
151
152
|
end
|
152
153
|
|
@@ -3,6 +3,7 @@ class RemoteStep
|
|
3
3
|
attr_accessor :override_dependencies
|
4
4
|
|
5
5
|
def init_job(cache_type = nil, other_params = {})
|
6
|
+
return self if @url
|
6
7
|
cache_type = :asynchronous if cache_type.nil? and not @is_exec
|
7
8
|
cache_type = :exec if cache_type.nil?
|
8
9
|
@last_info_time = nil
|
@@ -10,20 +11,17 @@ class RemoteStep
|
|
10
11
|
@server, @server_path = RemoteWorkflow::SSH.parse_url base_url
|
11
12
|
@input_id ||= "inputs-" << rand(100000).to_s
|
12
13
|
|
13
|
-
if override_dependencies
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
name, _sep, value = od.partition("=")
|
18
|
-
inputs[name] = value
|
19
|
-
end
|
14
|
+
if override_dependencies && override_dependencies.any?
|
15
|
+
override_dependencies.each do |od|
|
16
|
+
name, _sep, value = od.partition("=")
|
17
|
+
inputs[name] = value
|
20
18
|
end
|
21
|
-
|
22
|
-
RemoteWorkflow::SSH.upload_inputs(@server, inputs, @input_types, @input_id)
|
23
|
-
else
|
24
|
-
RemoteWorkflow::SSH.upload_inputs(@server, inputs, @input_types, @input_id)
|
25
19
|
end
|
26
20
|
|
21
|
+
inputs.select{|i| Step === i }.each{|i| i.produce }
|
22
|
+
|
23
|
+
RemoteWorkflow::SSH.upload_inputs(@server, inputs, @input_types, @input_id)
|
24
|
+
|
27
25
|
@name ||= Persist.memory("RemoteSteps", :workflow => self, :task => task, :jobname => @name, :inputs => inputs, :cache_type => cache_type) do
|
28
26
|
Misc.insist do
|
29
27
|
input_types = {}
|
@@ -51,8 +51,8 @@ module Workflow
|
|
51
51
|
when :path
|
52
52
|
inputs[input.to_sym] = Open.read(file).strip.split("\n").first
|
53
53
|
when :io
|
54
|
-
inputs[input.to_sym] = Open.open(Open.realpath(file))
|
55
|
-
when :
|
54
|
+
inputs[input.to_sym] = Open.open(Open.realpath(file))
|
55
|
+
when :io_array
|
56
56
|
inputs[input.to_sym] = Open.realpath(file).split("\n").collect{|f| Open.open(f)}
|
57
57
|
when :step_array
|
58
58
|
steps = Open.read(file).strip.split("\n").collect{|path| Workflow.load_step(path) }
|
@@ -97,6 +97,10 @@ module Workflow
|
|
97
97
|
inputs[input.to_sym] = TSV.open(file)
|
98
98
|
when :boolean
|
99
99
|
inputs[input.to_sym] = (file.read.strip == 'true')
|
100
|
+
when :integer
|
101
|
+
inputs[input.to_sym] = file.read.to_i
|
102
|
+
when :float
|
103
|
+
inputs[input.to_sym] = file.read.to_f
|
100
104
|
else
|
101
105
|
Log.debug "Loading #{ input } from #{file}"
|
102
106
|
inputs[input.to_sym] = file.read.strip
|
@@ -172,10 +176,17 @@ class Step
|
|
172
176
|
end
|
173
177
|
|
174
178
|
Log.debug "Saving job input #{name} (#{type}) into #{path}"
|
175
|
-
|
179
|
+
|
180
|
+
if IO === value && value.respond_to?(:filename) && value.filename
|
181
|
+
Open.write(path, value.filename)
|
182
|
+
elsif IO === value
|
183
|
+
Open.write(path, value)
|
184
|
+
else
|
185
|
+
Open.write(path, value.to_s)
|
186
|
+
end
|
176
187
|
end
|
177
188
|
|
178
|
-
def self.save_inputs(inputs, input_types,
|
189
|
+
def self.save_inputs(inputs, input_types, dir)
|
179
190
|
inputs.each do |name,value|
|
180
191
|
type = input_types[name]
|
181
192
|
type = type.to_s if type
|
@@ -220,7 +231,7 @@ class Step
|
|
220
231
|
input_types = IndiferentHash.setup(input_types.merge(:override_dependencies => :array))
|
221
232
|
end
|
222
233
|
|
223
|
-
save_inputs(inputs, input_types,
|
234
|
+
save_inputs(inputs, input_types, dir)
|
224
235
|
|
225
236
|
inputs.keys
|
226
237
|
end
|
@@ -75,6 +75,7 @@ class Step
|
|
75
75
|
|
76
76
|
def dirty?
|
77
77
|
return true if Open.exists?(pid_file) && ! ( Open.exists?(info_file) || done? )
|
78
|
+
return true if done? && ! (status == :done || status == :noinfo)
|
78
79
|
return false unless done? || status == :done
|
79
80
|
return false unless ENV["RBBT_UPDATE"] == "true"
|
80
81
|
|
data/lib/rbbt/workflow/step.rb
CHANGED
@@ -184,9 +184,13 @@ puts files * "\n"
|
|
184
184
|
|
185
185
|
target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
|
186
186
|
|
187
|
+
target_path = File.join(target, *path.split("/")[-3..-1])
|
188
|
+
|
187
189
|
subpath_files.each do |subpath, files|
|
188
190
|
Rbbt.migrate_files([subpath], target, options.merge(:files => files))
|
189
191
|
end
|
192
|
+
|
193
|
+
target_path
|
190
194
|
end
|
191
195
|
|
192
196
|
def self.purge(path, recursive = false, skip_overriden = true)
|
data/share/Rlib/util.R
CHANGED
@@ -232,20 +232,26 @@ rbbt.tsv2tibble <- function(data){
|
|
232
232
|
as_tibble(data, rownames=attr(data, 'key.field'))
|
233
233
|
}
|
234
234
|
|
235
|
-
rbbt.tsv.write <- function(filename, data, key.field = NULL, extra_headers = NULL, eol="\n", ...){
|
235
|
+
rbbt.tsv.write <- function(filename, data, key.field = NULL, extra_headers = NULL, eol="\n", type = 'list', names = NULL, ...){
|
236
236
|
|
237
237
|
if (is.null(key.field)){ key.field = attributes(data)$key.field;}
|
238
238
|
if (is.null(key.field)){ key.field = "ID";}
|
239
239
|
|
240
240
|
f = file(filename, 'wb');
|
241
241
|
|
242
|
+
if (is.null(extra_headers)){
|
243
|
+
extra_headers = paste(":type",type,sep="=")
|
244
|
+
}
|
245
|
+
|
242
246
|
if (!is.null(extra_headers)){
|
243
247
|
extra_headers = paste("#: ", extra_headers, "\n", sep="");
|
244
248
|
cat(extra_headers, file=f);
|
245
249
|
}
|
246
250
|
|
247
251
|
header = paste("#", key.field, sep="");
|
248
|
-
|
252
|
+
if (is.null(names)){ names = colnames(data)}
|
253
|
+
if (is.null(names)){ names = names(data)}
|
254
|
+
for (name in names){ header = paste(header, name, sep="\t");}
|
249
255
|
header = paste(header, "\n", sep="");
|
250
256
|
cat(header, file=f);
|
251
257
|
|
@@ -96,6 +96,8 @@ Misc.in_dir(app_dir) do
|
|
96
96
|
system ENV, "env RBBT_LOG=0 passenger start -R '#{config_ru_file}' -p #{options[:Port] || "2887"}"
|
97
97
|
when 'puma_alt'
|
98
98
|
system ENV, "puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 3 -t 8:32 --preload"
|
99
|
+
when 'puma_production'
|
100
|
+
CMD.cmd_log("env RBBT_LOG=#{Log.severity.to_s} puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 20 -t 10:160 --preload")
|
99
101
|
else
|
100
102
|
options[:config] = config_ru_file
|
101
103
|
options[:threads] = "8:8"
|
@@ -0,0 +1,77 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rbbt-util'
|
4
|
+
require 'rbbt/util/simpleopt'
|
5
|
+
|
6
|
+
$0 = "rbbt #{$previous_commands*" "} #{ File.basename(__FILE__) }" if $previous_commands
|
7
|
+
|
8
|
+
options = SOPT.setup <<EOF
|
9
|
+
|
10
|
+
View TSV nicer
|
11
|
+
|
12
|
+
$ #{$0} [options] <filename.tsv|->
|
13
|
+
|
14
|
+
Use - to read from STDIN
|
15
|
+
|
16
|
+
-h--help Print this help
|
17
|
+
|
18
|
+
EOF
|
19
|
+
if options[:help]
|
20
|
+
if defined? rbbt_usage
|
21
|
+
rbbt_usage
|
22
|
+
else
|
23
|
+
puts SOPT.doc
|
24
|
+
end
|
25
|
+
exit 0
|
26
|
+
end
|
27
|
+
|
28
|
+
|
29
|
+
file = ARGV.shift
|
30
|
+
|
31
|
+
file = STDIN if file == '-'
|
32
|
+
|
33
|
+
case
|
34
|
+
when options[:tokyocabinet]
|
35
|
+
tsv = Persist.open_tokyocabinet(file, false)
|
36
|
+
when options[:tokyocabinet_bd]
|
37
|
+
tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
|
38
|
+
else
|
39
|
+
if String === file
|
40
|
+
file = file.dup
|
41
|
+
Path.setup(File.expand_path(file))
|
42
|
+
end
|
43
|
+
tsv = file
|
44
|
+
end
|
45
|
+
|
46
|
+
tsv = TSV.open(tsv) if IO === tsv
|
47
|
+
|
48
|
+
|
49
|
+
begin
|
50
|
+
require 'rbbt/workflow'
|
51
|
+
Workflow.require_workflow "Genomics"
|
52
|
+
tsv = Genomics.job(:names, nil, :tsv => tsv).run
|
53
|
+
rescue
|
54
|
+
end
|
55
|
+
|
56
|
+
puts [Log.highlight(Log.color(:magenta, tsv.key_field)), Log.color(:yellow, tsv.fields * "\t")] * "\t"
|
57
|
+
TSV.traverse tsv do |k,values,fields|
|
58
|
+
fvalues = values.zip(fields).collect do |v,field|
|
59
|
+
if String === v
|
60
|
+
v = v.to_f.round(2) if v =~ /^-?\d+\.\d+$/
|
61
|
+
v = v.to_i if v =~ /^-?\d+$/
|
62
|
+
end
|
63
|
+
|
64
|
+
if Numeric === v
|
65
|
+
if field =~ /p.?value/i
|
66
|
+
v = v.round(4)
|
67
|
+
v < 0.05 ? Log.color(:cyan, v.to_s) : v
|
68
|
+
else
|
69
|
+
v = v.round(2)
|
70
|
+
v < 0 ? Log.color(:red, v.to_s) : Log.color(:green, v.to_s)
|
71
|
+
end
|
72
|
+
else
|
73
|
+
v
|
74
|
+
end
|
75
|
+
end
|
76
|
+
puts [Log.color(:magenta, k), fvalues] * "\t"
|
77
|
+
end
|
data/test/rbbt/test_resource.rb
CHANGED
@@ -17,9 +17,18 @@ module TestResource
|
|
17
17
|
file 'foo' do |t|
|
18
18
|
Open.write(t.name, "TEST")
|
19
19
|
end
|
20
|
+
|
21
|
+
rule /.*/ do |t|
|
22
|
+
Open.write(t.name, "bar")
|
23
|
+
end
|
20
24
|
EOF
|
21
25
|
|
22
|
-
claim tmp.test.work.footest, :rake, tmp.test.rakefiles.foo
|
26
|
+
claim tmp.test.work.footest, :rake, TestResource.tmp.test.rakefiles.foo
|
27
|
+
|
28
|
+
claim tmp.test.work.file_proc, :file_proc do |file,filename|
|
29
|
+
Open.write(filename, file)
|
30
|
+
nil
|
31
|
+
end
|
23
32
|
end
|
24
33
|
|
25
34
|
class TestTSV < Test::Unit::TestCase
|
@@ -33,11 +42,12 @@ class TestTSV < Test::Unit::TestCase
|
|
33
42
|
assert TSV === TestResource.tmp.test.test_tsv.tsv
|
34
43
|
end
|
35
44
|
|
36
|
-
def
|
37
|
-
TestResource.tmp.test.work.footest.foo.read
|
38
|
-
|
45
|
+
def test_rake
|
46
|
+
assert_equal TestResource.tmp.test.work.footest.foo.read, "TEST"
|
47
|
+
assert_equal TestResource.tmp.test.work.footest.bar.read, "bar"
|
39
48
|
end
|
40
49
|
|
50
|
+
|
41
51
|
def test_proc
|
42
52
|
assert TestResource.tmp.test.proc.read == "PROC TEST"
|
43
53
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.34.
|
4
|
+
version: 5.34.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-07-
|
11
|
+
date: 2022-07-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -459,6 +459,7 @@ files:
|
|
459
459
|
- share/rbbt_commands/tsv/uncollapse
|
460
460
|
- share/rbbt_commands/tsv/unzip
|
461
461
|
- share/rbbt_commands/tsv/values
|
462
|
+
- share/rbbt_commands/tsv/view
|
462
463
|
- share/rbbt_commands/tsv/write_excel
|
463
464
|
- share/rbbt_commands/tsv/zip
|
464
465
|
- share/rbbt_commands/watch
|