rbbt-util 5.25.48 → 5.25.49
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/util/open.rb +1 -1
- data/share/rbbt_commands/workflow/trace +127 -67
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a8a3a98584959d0127d381ed0eae5b73aaa649fe
|
4
|
+
data.tar.gz: 6bf3a25808ba9fcae6d9040e39a102ad6c4c44c5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2966c9da9b0e33061c42a55832e9fd5fff3dafa4beab079ac4cbaae8ee22fc8f710f61669758342d6cfe26fb2a531829d3971f609d0b19fa0b1b6d915721ff81
|
7
|
+
data.tar.gz: d51d9e658746e68101ab971013c3ba54d5a0c9caf7e05ca0e5978bda83d959a7830b1afc9522fcf815f8c4d567bd2977f66e66ae36488e3a47c488716920bae5
|
data/lib/rbbt/util/open.rb
CHANGED
@@ -384,7 +384,7 @@ module Open
|
|
384
384
|
FileUtils.mkdir_p File.dirname(target) unless File.exist? File.dirname(target)
|
385
385
|
tmp_target = File.join(File.dirname(target), '.tmp_mv.' + File.basename(target))
|
386
386
|
FileUtils.cp source, tmp_target
|
387
|
-
FileUtils.
|
387
|
+
FileUtils.mv tmp_target, target
|
388
388
|
return
|
389
389
|
end
|
390
390
|
|
@@ -24,12 +24,13 @@ $ rbbt workflow trace <job-result> <file.png>
|
|
24
24
|
-w--width* Image Width
|
25
25
|
-h--height* Image Height
|
26
26
|
-s--size* Image Size (Height and Width)
|
27
|
+
-fg--fix_gap Remove execution gaps
|
27
28
|
EOF
|
28
29
|
|
29
30
|
SOPT.usage if options[:help]
|
30
31
|
|
31
32
|
file = ARGV.shift || "/home/mvazque2/.rbbt/var/jobs/Sample/haplotype/NA12878"
|
32
|
-
plot = ARGV.shift
|
33
|
+
plot = ARGV.shift
|
33
34
|
|
34
35
|
width, height, size = options.values_at :width, :height, :size
|
35
36
|
|
@@ -37,16 +38,13 @@ size = 800 if size.nil?
|
|
37
38
|
width = size if width.nil?
|
38
39
|
height = size if height.nil?
|
39
40
|
|
40
|
-
$seen = []
|
41
41
|
def get_step(file)
|
42
42
|
file = File.expand_path(file)
|
43
43
|
file = file.sub(/\.(info|files)/,'')
|
44
|
-
$seen << file
|
45
44
|
Workflow.load_step file
|
46
45
|
end
|
47
46
|
|
48
47
|
step = get_step file
|
49
|
-
$main_mtime = Open.exist?(step.path) ? Open.mtime(step.path) : nil
|
50
48
|
|
51
49
|
data = TSV.setup({}, "Job~Workflow,Task,Start,End#:type=:list")
|
52
50
|
|
@@ -60,7 +58,11 @@ jobs.each do |job|
|
|
60
58
|
next unless job.done?
|
61
59
|
started = job.info[:started]
|
62
60
|
ddone = job.info[:done]
|
63
|
-
|
61
|
+
|
62
|
+
code = [job.workflow, job.task_name].compact.collect{|s| s.to_s} * "."
|
63
|
+
code = code + '.' + job.clean_name
|
64
|
+
|
65
|
+
data[code] = [job.workflow.to_s, job.task_name,started,ddone]
|
64
66
|
if min_start.nil?
|
65
67
|
min_start = started
|
66
68
|
else
|
@@ -82,65 +84,123 @@ data.add_field "End.second" do |k,value|
|
|
82
84
|
value["End"] - min_start
|
83
85
|
end
|
84
86
|
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
data
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
87
|
+
if options[:fix_gap]
|
88
|
+
ranges = []
|
89
|
+
data.through do |k,values|
|
90
|
+
start, eend = values.values_at "Start.second", "End.second"
|
91
|
+
|
92
|
+
ranges << (start.floor..eend.ceil)
|
93
|
+
end
|
94
|
+
|
95
|
+
gaps = {}
|
96
|
+
last = nil
|
97
|
+
Misc.collapse_ranges(ranges).each do |range|
|
98
|
+
start = range.begin
|
99
|
+
eend = range.end
|
100
|
+
if last
|
101
|
+
gaps[last] = start - last
|
102
|
+
end
|
103
|
+
last = eend
|
104
|
+
end
|
105
|
+
|
106
|
+
data.process "End.second" do |value,k,values|
|
107
|
+
gap = Misc.sum(gaps.select{|pos,size| pos < values["Start.second"]}.collect{|pos,size| size})
|
108
|
+
value - gap
|
109
|
+
end
|
110
|
+
|
111
|
+
data.process "Start.second" do |value,k,values|
|
112
|
+
gap = Misc.sum(gaps.select{|pos,size| pos < values["Start.second"]}.collect{|pos,size| size})
|
113
|
+
value - gap
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
tasks_info = {}
|
118
|
+
|
119
|
+
jobs.each do |dep|
|
120
|
+
next unless dep.done?
|
121
|
+
task = [dep.workflow, dep.task_name].compact.collect{|s| s.to_s} * "#"
|
122
|
+
info = tasks_info[task] ||= {}
|
123
|
+
|
124
|
+
time = dep.info[:done] - dep.info[:started]
|
125
|
+
info[:time] ||= []
|
126
|
+
info[:time] << time
|
127
|
+
|
128
|
+
cpus = nil
|
129
|
+
spark = false
|
130
|
+
shard = false
|
131
|
+
dep.info[:config_keys].select do |kinfo|
|
132
|
+
key, value, tokens = kinfo
|
133
|
+
key = key.to_s
|
134
|
+
cpus = value if key.include? 'cpu'
|
135
|
+
spark = value if key == 'spark'
|
136
|
+
shard = value if key == 'shard'
|
137
|
+
end
|
138
|
+
|
139
|
+
info[:cpus] = cpus || 1
|
140
|
+
info[:spark] = spark
|
141
|
+
info[:shard] = shard
|
142
|
+
end
|
143
|
+
|
144
|
+
stats = TSV.setup({}, "Task~Calls,Time,Cpus,Spark,Shard#:type=:list")
|
145
|
+
|
146
|
+
tasks_info.each do |task, info|
|
147
|
+
time_lists, cpus, spark, shard = info.values_at :time, :cpus, :spark, :shard
|
148
|
+
avg_time = Misc.mean(time_lists).to_i
|
149
|
+
calls = time_lists.length
|
150
|
+
stats[task] = [calls,avg_time, cpus, spark, shard]
|
151
|
+
end
|
152
|
+
|
153
|
+
puts stats.to_s
|
154
|
+
|
155
|
+
if plot
|
156
|
+
data.R <<-EOF, [:svg]
|
157
|
+
rbbt.require('tidyverse')
|
158
|
+
rbbt.require('ggplot2')
|
159
|
+
|
160
|
+
names(data) <- make.names(names(data))
|
161
|
+
data$id = rownames(data)
|
162
|
+
data$content = data$Task
|
163
|
+
data$start = data$Start
|
164
|
+
data$end = data$End
|
165
|
+
data$Project = data$Workflow
|
166
|
+
|
167
|
+
tasks = data
|
168
|
+
|
169
|
+
#theme_gantt <- function(base_size=11, base_family="Source Sans Pro Light") {
|
170
|
+
theme_gantt <- function(base_size=11, base_family="Sans Serif") {
|
171
|
+
ret <- theme_bw(base_size, base_family) %+replace%
|
172
|
+
theme(panel.background = element_rect(fill="#ffffff", colour=NA),
|
173
|
+
axis.title.x=element_text(vjust=-0.2), axis.title.y=element_text(vjust=1.5),
|
174
|
+
title=element_text(vjust=1.2, family="Source Sans Pro Semibold"),
|
175
|
+
panel.border = element_blank(), axis.line=element_blank(),
|
176
|
+
panel.grid.minor=element_blank(),
|
177
|
+
panel.grid.major.y = element_blank(),
|
178
|
+
panel.grid.major.x = element_line(size=0.5, colour="grey80"),
|
179
|
+
axis.ticks=element_blank(),
|
180
|
+
legend.position="bottom",
|
181
|
+
axis.title=element_text(size=rel(1.2), family="Source Sans Pro Semibold"),
|
182
|
+
strip.text=element_text(size=rel(1.5), family="Source Sans Pro Semibold"),
|
183
|
+
strip.background=element_rect(fill="#ffffff", colour=NA),
|
184
|
+
panel.spacing.y=unit(1.5, "lines"),
|
185
|
+
legend.key = element_blank())
|
186
|
+
|
187
|
+
ret
|
188
|
+
}
|
189
|
+
|
190
|
+
tasks.long <- tasks %>%
|
191
|
+
gather(date.type, task.date, -c(Project, Task, id, Start.second, End.second)) %>%
|
192
|
+
arrange(date.type, task.date) %>%
|
193
|
+
mutate(id = factor(id, levels=rev(unique(id)), ordered=TRUE))
|
194
|
+
|
195
|
+
x.breaks <- seq(length(tasks$Task) + 0.5 - 3, 0, by=-3)
|
196
|
+
|
197
|
+
timeline <- ggplot(tasks.long, aes(y=id, yend=id, x=Start.second, xend=End.second, colour=Task)) +
|
198
|
+
geom_segment() +
|
199
|
+
geom_vline(xintercept=x.breaks, colour="grey80", linetype="dotted") +
|
200
|
+
guides(colour=guide_legend(title=NULL)) +
|
201
|
+
labs(x=NULL, y=NULL) +
|
202
|
+
theme_gantt() + theme(axis.text.x=element_text(angle=45, hjust=1))
|
203
|
+
|
204
|
+
rbbt.png_plot('#{plot}', 'timeline', width=#{width}, height=#{height}, pointsize=6)
|
205
|
+
EOF
|
206
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.25.
|
4
|
+
version: 5.25.49
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-04-
|
11
|
+
date: 2019-04-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|