rbbt-util 5.25.48 → 5.25.49

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 46222cc21bab86e8e9d9783431a1607aeb2b3a8d
-  data.tar.gz: c9912dfa07e5bef78b599ef53cf765033d5fa098
+  metadata.gz: a8a3a98584959d0127d381ed0eae5b73aaa649fe
+  data.tar.gz: 6bf3a25808ba9fcae6d9040e39a102ad6c4c44c5
 SHA512:
-  metadata.gz: fe49673ea95fc2d2715459070c1c8e77af47b0281560b4663251980daf074b642c3077ff8c8b35609eeadbff35f78299abcb149b556c53e5fcf3c1ef6963090c
-  data.tar.gz: 10fc19079bd5de99f8b21e5ef2b23435a9a2452c99f82c95c98c6c6810947853795fd0250ae6172a0de0bda04d9fe9daefe708dc9f2d1b6974141b23c33f7f8f
+  metadata.gz: 2966c9da9b0e33061c42a55832e9fd5fff3dafa4beab079ac4cbaae8ee22fc8f710f61669758342d6cfe26fb2a531829d3971f609d0b19fa0b1b6d915721ff81
+  data.tar.gz: d51d9e658746e68101ab971013c3ba54d5a0c9caf7e05ca0e5978bda83d959a7830b1afc9522fcf815f8c4d567bd2977f66e66ae36488e3a47c488716920bae5

data/lib/rbbt/util/open.rb

@@ -384,7 +384,7 @@ module Open
       FileUtils.mkdir_p File.dirname(target) unless File.exist? File.dirname(target)
       tmp_target = File.join(File.dirname(target), '.tmp_mv.' + File.basename(target))
       FileUtils.cp source, tmp_target
-      FileUtils.cp tmp_target, target
+      FileUtils.mv tmp_target, target
       return
     end
 

data/share/rbbt_commands/workflow/trace

@@ -24,12 +24,13 @@ $ rbbt workflow trace <job-result> <file.png>
 -w--width* Image Width
 -h--height* Image Height
 -s--size* Image Size (Height and Width)
+-fg--fix_gap Remove execution gaps
 EOF
 
 SOPT.usage if options[:help]
 
 file = ARGV.shift || "/home/mvazque2/.rbbt/var/jobs/Sample/haplotype/NA12878"
-plot = ARGV.shift || "/tmp/plot.png"
+plot = ARGV.shift 
 
 width, height, size = options.values_at :width, :height, :size
 
@@ -37,16 +38,13 @@ size = 800 if size.nil?
 width = size if width.nil?
 height = size if height.nil?
 
-$seen = []
 def get_step(file)
   file = File.expand_path(file)
   file = file.sub(/\.(info|files)/,'')
-  $seen << file
   Workflow.load_step file
 end
 
 step = get_step file
-$main_mtime = Open.exist?(step.path) ? Open.mtime(step.path) : nil
 
 data = TSV.setup({}, "Job~Workflow,Task,Start,End#:type=:list")
 
@@ -60,7 +58,11 @@ jobs.each do |job|
   next unless job.done?
   started = job.info[:started]
   ddone = job.info[:done]
-  data[[job.task_name.to_s, job.clean_name] * "."] = [job.workflow.to_s, job.task_name,started,ddone]
+
+  code = [job.workflow, job.task_name].compact.collect{|s| s.to_s} * "."
+  code = code + '.' + job.clean_name
+
+  data[code] = [job.workflow.to_s, job.task_name,started,ddone]
   if min_start.nil?
     min_start = started
   else
@@ -82,65 +84,123 @@ data.add_field "End.second" do |k,value|
   value["End"] - min_start
 end
 
-Log.severity = 0
-#data.R_interactive <<-EOF
-#  rbbt.require('timevis')
-#  data$id = rownames(data)
-#  data$content = data$Task
-#  data$start = data$Start
-#  data$end = data$End
-#
-#  timevis(data)
-#EOF
-
-
-data.R <<-EOF, [:svg]
-  rbbt.require('tidyverse')
-  rbbt.require('ggplot2')
-
-  names(data) <- make.names(names(data))
-  data$id = rownames(data)
-  data$content = data$Task
-  data$start = data$Start
-  data$end = data$End
-  data$Project = data$Workflow
-
-  tasks = data
-
-  #theme_gantt <- function(base_size=11, base_family="Source Sans Pro Light") {
-  theme_gantt <- function(base_size=11, base_family="Sans Serif") {
-    ret <- theme_bw(base_size, base_family) %+replace%
-      theme(panel.background = element_rect(fill="#ffffff", colour=NA),
-            axis.title.x=element_text(vjust=-0.2), axis.title.y=element_text(vjust=1.5),
-            title=element_text(vjust=1.2, family="Source Sans Pro Semibold"),
-            panel.border = element_blank(), axis.line=element_blank(),
-            panel.grid.minor=element_blank(),
-            panel.grid.major.y = element_blank(),
-            panel.grid.major.x = element_line(size=0.5, colour="grey80"),
-            axis.ticks=element_blank(),
-            legend.position="bottom", 
-            axis.title=element_text(size=rel(0.8), family="Source Sans Pro Semibold"),
-            strip.text=element_text(size=rel(1), family="Source Sans Pro Semibold"),
-            strip.background=element_rect(fill="#ffffff", colour=NA),
-            panel.spacing.y=unit(1.5, "lines"),
-            legend.key = element_blank())
-    
-    ret
-  }
-
-  tasks.long <- tasks %>%
-  gather(date.type, task.date, -c(Project, Task, id, Start.second, End.second)) %>%
-  arrange(date.type, task.date) %>%
-  mutate(id = factor(id, levels=rev(unique(id)), ordered=TRUE))
-
-  x.breaks <- seq(length(tasks$Task) + 0.5 - 3, 0, by=-3)
-
-  timeline <- ggplot(tasks.long, aes(y=id, yend=id, x=Start.second, xend=End.second, colour=Task)) + 
-    geom_segment() + 
-    geom_vline(xintercept=x.breaks, colour="grey80", linetype="dotted") + 
-    guides(colour=guide_legend(title=NULL)) +
-    labs(x=NULL, y=NULL) + 
-    theme_gantt() + theme(axis.text.x=element_text(angle=45, hjust=1))
-
-  rbbt.png_plot('#{plot}', 'timeline', width=#{width}, height=#{height})
-EOF
+if options[:fix_gap]
+  ranges = []
+  data.through do |k,values|
+    start, eend = values.values_at "Start.second", "End.second"
+
+    ranges << (start.floor..eend.ceil)
+  end
+
+  gaps = {}
+  last = nil
+  Misc.collapse_ranges(ranges).each do |range|
+    start = range.begin
+    eend = range.end
+    if last
+      gaps[last] = start - last
+    end
+    last = eend
+  end
+
+  data.process "End.second" do |value,k,values|
+    gap = Misc.sum(gaps.select{|pos,size| pos < values["Start.second"]}.collect{|pos,size| size})
+    value - gap
+  end
+
+  data.process "Start.second" do |value,k,values|
+    gap = Misc.sum(gaps.select{|pos,size| pos < values["Start.second"]}.collect{|pos,size| size})
+    value - gap
+  end
+end
+
+tasks_info = {}
+
+jobs.each do |dep|
+  next unless dep.done?
+  task = [dep.workflow, dep.task_name].compact.collect{|s| s.to_s} * "#"
+  info = tasks_info[task] ||= {}
+
+  time = dep.info[:done] - dep.info[:started]
+  info[:time] ||= []
+  info[:time] << time
+
+  cpus = nil
+  spark = false
+  shard = false
+  dep.info[:config_keys].select do |kinfo| 
+    key, value, tokens = kinfo
+    key = key.to_s
+    cpus = value if key.include? 'cpu'
+    spark = value if key == 'spark'
+    shard = value if key == 'shard'
+  end
+
+  info[:cpus] = cpus || 1
+  info[:spark] = spark
+  info[:shard] = shard
+end
+
+stats = TSV.setup({}, "Task~Calls,Time,Cpus,Spark,Shard#:type=:list")
+
+tasks_info.each do |task, info|
+  time_lists, cpus, spark, shard = info.values_at :time, :cpus, :spark, :shard
+  avg_time = Misc.mean(time_lists).to_i
+  calls = time_lists.length
+  stats[task] = [calls,avg_time, cpus, spark, shard]
+end
+
+puts stats.to_s
+
+if plot
+  data.R <<-EOF, [:svg]
+    rbbt.require('tidyverse')
+    rbbt.require('ggplot2')
+
+    names(data) <- make.names(names(data))
+    data$id = rownames(data)
+    data$content = data$Task
+    data$start = data$Start
+    data$end = data$End
+    data$Project = data$Workflow
+
+    tasks = data
+
+    #theme_gantt <- function(base_size=11, base_family="Source Sans Pro Light") {
+    theme_gantt <- function(base_size=11, base_family="Sans Serif") {
+      ret <- theme_bw(base_size, base_family) %+replace%
+        theme(panel.background = element_rect(fill="#ffffff", colour=NA),
+              axis.title.x=element_text(vjust=-0.2), axis.title.y=element_text(vjust=1.5),
+              title=element_text(vjust=1.2, family="Source Sans Pro Semibold"),
+              panel.border = element_blank(), axis.line=element_blank(),
+              panel.grid.minor=element_blank(),
+              panel.grid.major.y = element_blank(),
+              panel.grid.major.x = element_line(size=0.5, colour="grey80"),
+              axis.ticks=element_blank(),
+              legend.position="bottom", 
+              axis.title=element_text(size=rel(1.2), family="Source Sans Pro Semibold"),
+              strip.text=element_text(size=rel(1.5), family="Source Sans Pro Semibold"),
+              strip.background=element_rect(fill="#ffffff", colour=NA),
+              panel.spacing.y=unit(1.5, "lines"),
+              legend.key = element_blank())
+
+      ret
+    }
+
+    tasks.long <- tasks %>%
+    gather(date.type, task.date, -c(Project, Task, id, Start.second, End.second)) %>%
+    arrange(date.type, task.date) %>%
+    mutate(id = factor(id, levels=rev(unique(id)), ordered=TRUE))
+
+    x.breaks <- seq(length(tasks$Task) + 0.5 - 3, 0, by=-3)
+
+    timeline <- ggplot(tasks.long, aes(y=id, yend=id, x=Start.second, xend=End.second, colour=Task)) + 
+      geom_segment() + 
+      geom_vline(xintercept=x.breaks, colour="grey80", linetype="dotted") + 
+      guides(colour=guide_legend(title=NULL)) +
+      labs(x=NULL, y=NULL) + 
+      theme_gantt() + theme(axis.text.x=element_text(angle=45, hjust=1))
+
+    rbbt.png_plot('#{plot}', 'timeline', width=#{width}, height=#{height}, pointsize=6)
+  EOF
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.25.48
+  version: 5.25.49
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-14 00:00:00.000000000 Z
+date: 2019-04-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake