rbbt-util 5.5.68 → 5.6.0

data/lib/rbbt/util/simpleopt.rb

@@ -95,7 +95,7 @@ module SOPT
   def self.parse(opts)
     info = {}
 
-    opts.split(/:/).each do |opt|
+    opts.split(/[:\n]+/).each do |opt|
       next if opt.strip.empty?
 
       short, long = opt.strip.sub(/(^[^\s]*)\*/,'\1').split('--').values_at(0,1)

data/lib/rbbt/workflow.rb

@@ -196,9 +196,13 @@ module Workflow
 
     dependencies = real_dependencies(task, jobname, inputs, task_dependencies[taskname] || [])
 
-    input_values = task.take_input_values(inputs)
-
-    step_path = step_path taskname, jobname, input_values, dependencies
+    if inputs.empty?
+      step_path = step_path taskname, jobname, [], [], task.extension
+      input_values = task.take_input_values(inputs)
+    else
+      input_values = task.take_input_values(inputs)
+      step_path = step_path taskname, jobname, input_values, dependencies, task.extension
+    end
 
     step = Step.new step_path, task, input_values, dependencies
 
@@ -258,4 +262,14 @@ module Workflow
     local_persist_setup
     local_workdir_setup
   end
+
+  def with_workdir(workdir)
+    saved = self.workdir
+    begin
+      self.workdir = Path.setup(File.expand_path(workdir))
+      yield
+    ensure
+      self.workdir = saved
+    end
+  end
 end

data/lib/rbbt/workflow/accessor.rb

@@ -281,7 +281,7 @@ module Workflow
   end
 
   TAG = :hash
-  def step_path(taskname, jobname, inputs, dependencies)
+  def step_path(taskname, jobname, inputs, dependencies, extension = nil)
     Proc.new{
       raise "Jobname makes an invalid path: #{ jobname }" if jobname =~ /\.\./
       if inputs.any? or dependencies.any?
@@ -295,6 +295,10 @@ module Workflow
         tagged_jobname = jobname
       end
 
+      if extension and not extension.empty?
+        tagged_jobname = tagged_jobname + ('.' << extension.to_s)
+      end
+
       workdir[taskname][tagged_jobname].find
     }
   end

data/lib/rbbt/workflow/definition.rb

@@ -7,6 +7,7 @@ module Workflow
   AnnotatedModule.add_consummable_annotation(self,
                                              :result_description => "",
                                              :result_type        => nil,
+                                             :extension          => '',
                                              :dependencies       => [])
   def helper(name, &block)
     helpers[name] = block
@@ -16,6 +17,10 @@ module Workflow
     @description = description
   end
 
+  def extension(extension)
+    @extension = extension
+  end
+
   def returns(description)
     @result_description = description
   end
@@ -45,6 +50,7 @@ module Workflow
       :result_type        => (Array === type ? type.to_sym : type),
       :input_defaults     => consume_input_defaults,
       :input_descriptions => consume_input_descriptions,
+      :extension          => consume_extension,
       :input_options      => consume_input_options
     }
     

data/lib/rbbt/workflow/step.rb

@@ -92,12 +92,18 @@ class Step
     self
   end
 
+  def checks
+    deps = rec_dependencies.collect{|dependency| dependency.path }.uniq
+    deps.select!{|p| p.exists? }
+  end
+
   def run(no_load = false)
-    result = Persist.persist "Job", @task.result_type, :file => path, :check => rec_dependencies.collect{|dependency| dependency.path }.uniq, :no_load => no_load do
-      @exec = false
+
+    result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load do
       if Step === Step.log_relay_step and not self == Step.log_relay_step
         relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
       end
+      @exec = false
 
       Open.rm info_file if Open.exists? info_file
 
@@ -149,7 +155,7 @@ class Step
               backtrace = $!.backtrace
 
               # HACK: This fixes an strange behaviour in 1.9.3 where some
-              # bactrace strings are coded in ASCII-8BIT
+              # backtrace strings are coded in ASCII-8BIT
               backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
 
               set_info :backtrace, backtrace 
@@ -251,7 +257,7 @@ class Step
 
   def load
     raise "Can not load: Step is waiting for proces #{@pid} to finish" if not done?
-    result = Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path} do
+    result = Persist.persist "Job", @task.result_type, :file => @path, :check => checks do
       exec
     end
     prepare_result result, @task.result_description, info

data/lib/rbbt/workflow/task.rb

@@ -2,7 +2,7 @@ require 'rbbt/util/misc'
 require 'rbbt/persist'
 
 module Task
-  attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :input_options, :description, :name, :result_description
+  attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :input_options, :description, :name, :result_description, :extension
 
   def self.setup(options = {}, &block)
     block.extend Task

data/share/rbbt_commands/tsv/json

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+options = SOPT.get("-tch--tokyocabinet:-tcb--tokyocabinet_bd:-m--melt:-s--snake_case")
+
+file = ARGV.shift
+
+case
+when options[:tokyocabinet]
+  tsv = Persist.open_tokyocabinet(file, false)
+when options[:tokyocabinet_bd]
+  tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
+else
+  tsv = TSV.open(Open.open(file))
+end
+
+fields = tsv.fields
+
+if options[:snake_case]
+ tsv.fields = fields = fields.collect{|f| Misc.snake_case f}
+end
+
+raise "No fields specified" if fields.nil?
+if options[:melt]
+ puts tsv.to_hash.collect{|key, values|
+  h = {:key => key}
+  fields.zip(values).each do |field, v|
+   h[field] = v
+  end
+  h
+ }.to_json
+else
+ puts tsv.to_hash.to_json
+end
+

data/share/rbbt_commands/workflow/task

@@ -112,6 +112,8 @@ options = SOPT.get <<EOF
 -rcl--recursive_clean Clean the last step and its dependencies to recompute the job completely:
 -jn--jobname* Job name to use. The name 'Default' is used by default:
 -pn--printname Print the name of the job and exit without starting it:
+-wd--workdir* Change the working directory of the workflow:
+-O--output* Save job result into file:
 --fork Run job asyncronously:
 EOF
 
@@ -169,6 +171,8 @@ sopt_option_string = SOPT_options(workflow, task)
 job_options = SOPT.get sopt_option_string
 job_options = fix_options(workflow, task, job_options)
 
+workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir))) if options[:workdir]
+
 #- get job
 
 job = workflow.job(task.name, name, job_options)
@@ -226,10 +230,12 @@ rescue ParameterException
     usage(workflow, task, $!.message)
 end
 
+out = options.include?(:output) ? File.open(options[:output], 'wb') : STDOUT
+
 if Step === res
-  puts Open.read(res.path) if File.exists? res.path
+  out.puts Open.read(res.path) if File.exists? res.path
 else
-  puts res
+  out.puts res
 end
 
 exit 0

data/test/rbbt/persist/test_tsv.rb

@@ -0,0 +1,77 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
+require 'rbbt/persist'
+require 'rbbt/annotations'
+require 'rbbt/util/tmpfile'
+require 'test/unit'
+
+
+
+class TestPersistTSV < Test::Unit::TestCase
+  MAX = 100_002
+
+  def tsv_path
+    Random.new
+    #require 'rbbt/sources/organism'
+    #Organism.identifiers("Hsa")
+    require 'rbbt/workflow'
+    Workflow.require_workflow "Genomics"
+    require 'rbbt/entity/gene'
+    require 'rbbt/sources/genomes1000'
+    CMD.cmd("grep -v '^\\.\t' | head -n #{MAX}", :in => Genomes1000.mutations.open, :pipe => true)
+  end
+
+  def run_bechmark(file, engine)
+    Log.info "Testing #{ Term::ANSIColor.red(engine) }"
+    TmpFile.with_file nil, false do |tmp_file|
+      db= nil
+      Misc.benchmark(1, "Build database with #{MAX - 2} entries") do
+        db = TSV.open(file, :fields => [1], :persist => true, :persist_engine => engine, :persist_dir => tmp_file, :type => :single, :unnamed => true)
+        fff db
+      end
+      test = db.keys.sort{rand}[1..100000]
+      Misc.benchmark(5, "Access #{test.length} random entries") do
+        test.each do |k| db[k] end
+      end
+      Log.info "Profiling access to #{test.length} random entries"
+      Misc.profile :min_percent => 0.1 do
+        test.each do |k| db[k] end
+      end
+      assert_equal "1:10611:G", db["rs189107123"]
+    end
+  end
+
+  def test_benchmark_tch
+    engine = "HDB"
+    run_bechmark(tsv_path, engine)
+  end
+
+  def test_benchmark_tcb
+    engine = "BDB"
+    run_bechmark(tsv_path, engine)
+  end
+
+  def test_benchmark_kch
+    engine = "kch"
+    run_bechmark(tsv_path, engine)
+  end
+
+  def test_benchmark_kcb
+    engine = "kct"
+    run_bechmark(tsv_path, engine)
+  end
+
+  def test_benchmark_cdb
+    engine = "CDB"
+    run_bechmark(tsv_path, engine)
+  end
+
+  def test_benchmark_leveldb
+    engine = "LevelDB"
+    run_bechmark(tsv_path, engine)
+  end
+
+  def __test_benchmark_lmdb
+    engine = "LMDB"
+    run_bechmark(tsv_path, engine)
+  end
+end

data/test/rbbt/persist/tsv/test_cdb.rb

@@ -0,0 +1,23 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../../test_helper')
+require 'rbbt/persist'
+require 'rbbt/annotations'
+require 'rbbt/util/tmpfile'
+require 'test/unit'
+
+module TestAnnotation
+  extend Annotation
+
+  self.annotation :test_annotation
+end
+
+class TestPersistTSVCDB < Test::Unit::TestCase
+
+  def test_organism
+    require 'rbbt/sources/organism'
+    TmpFile.with_file nil do |tmp_file|
+      file = CMD.cmd("head -n 1000000", :in => Organism.identifiers("Hsa").open, :pipe => true)
+      tsv = Organism.identifiers("Hsa").tsv(:key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "CDB", :persist_dir => tmp_file)
+      assert_equal "ENSG00000141510", tsv["TP53"]
+    end
+  end
+end

data/test/rbbt/persist/tsv/test_kyotocabinet.rb

@@ -0,0 +1,33 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../../test_helper')
+require 'rbbt/persist'
+require 'rbbt/persist/tsv'
+require 'rbbt/annotations'
+require 'rbbt/util/tmpfile'
+require 'test/unit'
+
+require 'rbbt/sources/organism'
+
+module TestAnnotation
+  extend Annotation
+
+  self.annotation :test_annotation
+end
+
+class TestPersistTSVKC < Test::Unit::TestCase
+  def _test_organism_kch
+    require 'rbbt/sources/organism'
+    TmpFile.with_file do |tmp_file|
+      tsv = Organism.identifiers("Hsa").tsv :key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "kch", :persist_dir => tmp_file
+      assert_equal "ENSG00000141510", tsv["TP53"]
+    end
+  end
+
+
+  def test_organism_kct
+    require 'rbbt/sources/organism'
+    TmpFile.with_file do |tmp_file|
+      tsv = Organism.identifiers("Hsa").tsv :key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "kct", :persist_dir => tmp_file
+      assert_equal "ENSG00000141510", tsv["TP53"]
+    end
+  end
+end

data/test/rbbt/persist/tsv/test_leveldb.rb

@@ -0,0 +1,22 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../../test_helper')
+require 'rbbt/persist'
+require 'rbbt/annotations'
+require 'rbbt/util/tmpfile'
+require 'test/unit'
+
+module TestAnnotation
+  extend Annotation
+
+  self.annotation :test_annotation
+end
+
+class TestPersistTSVLevelDB < Test::Unit::TestCase
+
+  def test_organism
+    require 'rbbt/sources/organism'
+    TmpFile.with_file do |tmp_file|
+      tsv = Organism.identifiers("Hsa").tsv :key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "LevelDB", :persist_dir => tmp_file
+      assert_equal "ENSG00000141510", tsv["TP53"]
+    end
+  end
+end

data/test/rbbt/persist/tsv/test_lmdb.rb

@@ -0,0 +1,22 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../../test_helper')
+require 'rbbt/persist'
+require 'rbbt/annotations'
+require 'rbbt/util/tmpfile'
+require 'test/unit'
+
+module TestAnnotation
+  extend Annotation
+
+  self.annotation :test_annotation
+end
+
+class TestPersistTSVLMDB < Test::Unit::TestCase
+
+  def test_organism
+    require 'rbbt/sources/organism'
+    TmpFile.with_file do |tmp_file|
+      tsv = Organism.identifiers("Hsa").tsv :key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "LMDB", :persist_dir => tmp_file
+      assert_equal "ENSG00000141510", tsv["TP53"]
+    end
+  end
+end

data/test/rbbt/persist/tsv/test_tokyocabinet.rb

@@ -0,0 +1,242 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../../test_helper')
+require 'rbbt/persist'
+require 'rbbt/annotations'
+require 'rbbt/util/tmpfile'
+require 'test/unit'
+
+module TestAnnotation
+  extend Annotation
+
+  self.annotation :test_annotation
+end
+
+class TestPersistTSVTC < Test::Unit::TestCase
+
+  def test_organism
+    require 'rbbt/sources/organism'
+    TmpFile.with_file do |tmp_file|
+      tsv = Organism.identifiers("Hsa").tsv :key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "HDB", :persist_dir => tmp_file
+      assert_equal "ENSG00000141510", tsv["TP53"]
+    end
+  end
+
+  def test_annotation_persist
+    TmpFile.with_file do |tmp|
+      entity1 = "Entity 1"
+      entity2 = "Entity 2"
+
+      TestAnnotation.setup(entity1, :test_annotation => "1")
+      TestAnnotation.setup(entity2, :test_annotation => "2")
+
+      annotations = [entity1, entity2]
+
+      persisted_annotations = Persist.persist("Test", :annotations, :file => tmp) do
+        annotations
+      end
+
+      assert_equal "Entity 1", persisted_annotations.first
+      assert_equal "Entity 2", persisted_annotations.last
+      assert_equal "1", persisted_annotations.first.test_annotation
+      assert_equal "2", persisted_annotations.last.test_annotation
+
+      persisted_annotations = Persist.persist("Test", :annotations, :file => tmp) do
+        annotations
+      end
+
+      assert_equal "Entity 1", persisted_annotations.sort.first
+      assert_equal "Entity 2", persisted_annotations.sort.last
+      assert_equal "1", persisted_annotations.sort.first.test_annotation
+      assert_equal "2", persisted_annotations.sort.last.test_annotation
+    end
+  end
+
+  def test_annotation_persist_with_repeitions
+    TmpFile.with_file do |tmp|
+      entity1 = "Entity 1"
+      entity2 = "Entity 2"
+      entity2bis = "Entity 2"
+
+      TestAnnotation.setup(entity1, :test_annotation => "1")
+      TestAnnotation.setup(entity2, :test_annotation => "2")
+      TestAnnotation.setup(entity2bis, :test_annotation => "2")
+
+      annotations = [entity1, entity2, entity2bis]
+
+      persisted_annotations = Persist.persist("Test", :annotations, :file => tmp) do
+        annotations
+      end
+
+      assert_equal 3, persisted_annotations.length
+
+      assert_equal "Entity 1", persisted_annotations.first
+      assert_equal "Entity 2", persisted_annotations.last
+      assert_equal "1", persisted_annotations.first.test_annotation
+      assert_equal "2", persisted_annotations.last.test_annotation
+
+      persisted_annotations = Persist.persist("Test", :annotations, :file => tmp) do
+        annotations
+      end
+
+      assert_equal 3, persisted_annotations.length
+
+      assert_equal "Entity 1", persisted_annotations.sort.first
+      assert_equal "Entity 2", persisted_annotations.sort.last
+      assert_equal "1", persisted_annotations.sort.first.test_annotation
+      assert_equal "2", persisted_annotations.sort.last.test_annotation
+    end
+  end
+
+  def test_bdb
+    TmpFile.with_file do |tmp|
+      repo = Persist.open_tokyocabinet(tmp, true, :double, TokyoCabinet::BDB)
+      repo["test:string1"] = [["STR1"]]
+      repo["test:string2"] = [["STR2"]]
+      repo["other_test:string3"] = [["STR2"]]
+
+      assert_equal ["test:string1", "test:string2"].sort, repo.range("test:" << 0.chr, false, "test:" << 255.chr, false).sort
+      assert_equal ["other_test:string3"].sort, repo.range("other_test:" << 0.chr, false, "other_test:" << 255.chr, false).sort
+    end
+  end
+
+  def test_annotation_persist_repo
+    TmpFile.with_file do |tmp|
+      repo = Persist.open_tokyocabinet(tmp, true, :list, TokyoCabinet::BDB)
+
+      entity1 = "Entity 1"
+      entity2 = "Entity 2"
+
+      TestAnnotation.setup(entity1, :test_annotation => "1")
+      TestAnnotation.setup(entity2, :test_annotation => "2")
+
+      annotations = [entity1, entity2]
+
+      persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
+        annotations
+      end
+
+      assert_equal "Entity 1", persisted_annotations.first
+      assert_equal "Entity 2", persisted_annotations.last
+      assert_equal "1", persisted_annotations.first.test_annotation
+      assert_equal "2", persisted_annotations.last.test_annotation
+
+      persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
+        annotations
+      end
+
+      assert_equal "Entity 1", persisted_annotations.sort.first
+      assert_equal "Entity 2", persisted_annotations.sort.last
+      assert_equal "1", persisted_annotations.sort.first.test_annotation
+      assert_equal "2", persisted_annotations.sort.last.test_annotation
+    end
+  end
+
+  def test_annotation_persist_repo_annotated_array
+    TmpFile.with_file do |tmp|
+      repo = Persist.open_tokyocabinet(tmp, true, :list, TokyoCabinet::BDB)
+
+      entity1 = "Entity 1"
+      entity2 = "Entity 2"
+
+      annotations = [entity1, entity2]
+      TestAnnotation.setup(annotations, :test_annotation => "1")
+      annotations.extend AnnotatedArray
+
+      persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
+        annotations
+      end
+
+      assert_equal "Entity 1", persisted_annotations.first
+      assert_equal "Entity 2", persisted_annotations.last
+      assert_equal "1", persisted_annotations.first.test_annotation
+      assert_equal "1", persisted_annotations.last.test_annotation
+
+      persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
+        annotations
+      end
+
+      persisted_annotations.extend AnnotatedArray
+
+      assert_equal "Entity 1", persisted_annotations.sort.first
+      assert_equal "Entity 2", persisted_annotations.sort.last
+      assert_equal "1", persisted_annotations.sort.first.test_annotation
+      assert_equal "1", persisted_annotations.sort.last.test_annotation
+    end
+  end
+
+  def test_annotation_persist_repo_triple_array
+    TmpFile.with_file(nil, false) do |tmp|
+      repo = Persist.open_database(tmp, true, :list, "BDB")
+
+      entity1 = "Entity 1"
+      entity2 = "Entity 2"
+
+      annotations = [entity1, entity2]
+      TestAnnotation.setup(annotations, :test_annotation => "1")
+      annotations.extend AnnotatedArray
+
+      annotations_ary = [annotations]
+      TestAnnotation.setup(annotations_ary, :test_annotation => "1")
+      annotations_ary.extend AnnotatedArray
+
+      persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
+        annotations_ary
+      end
+      assert AnnotatedArray === persisted_annotations
+
+      assert_equal "Entity 1", persisted_annotations.first.first
+      assert_equal "Entity 2", persisted_annotations.first.last
+      assert_equal "1", persisted_annotations.first.first.test_annotation
+      assert_equal "1", persisted_annotations.first.last.test_annotation
+
+      persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
+        annotations_ary
+      end
+
+      assert AnnotatedArray === persisted_annotations
+
+      assert_equal "Entity 1", persisted_annotations.sort.first.first
+      assert_equal "Entity 2", persisted_annotations.sort.first.last
+      assert_equal "1", persisted_annotations.sort.first.first.test_annotation
+      assert_equal "1", persisted_annotations.sort.first.last.test_annotation
+
+    end
+  end
+
+  def test_annotation_persist_repo_with_repetitions
+    TmpFile.with_file do |tmp|
+      repo = Persist.open_tokyocabinet(tmp, true, :list, TokyoCabinet::BDB)
+
+      entity1 = "Entity 1"
+      entity2 = "Entity 2"
+      entity2bis = "Entity 2"
+
+      TestAnnotation.setup(entity1, :test_annotation => "1")
+      TestAnnotation.setup(entity2, :test_annotation => "2")
+      TestAnnotation.setup(entity2bis, :test_annotation => "2")
+
+      annotations = [entity1, entity2, entity2bis]
+
+      persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
+        annotations
+      end
+
+      assert_equal 3, persisted_annotations.length
+
+      assert_equal "Entity 1", persisted_annotations.first
+      assert_equal "Entity 2", persisted_annotations.last
+      assert_equal "1", persisted_annotations.first.test_annotation
+      assert_equal "2", persisted_annotations.last.test_annotation
+
+      persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
+        annotations
+      end
+
+      assert_equal 3, persisted_annotations.length
+
+      assert_equal "Entity 1", persisted_annotations.sort.first
+      assert_equal "Entity 2", persisted_annotations.sort.last
+      assert_equal "1", persisted_annotations.sort.first.test_annotation
+      assert_equal "2", persisted_annotations.sort.last.test_annotation
+    end
+  end
+end