rbbt-util 2.0.1 → 2.1.0

data/bin/tsv.rb

@@ -2,7 +2,7 @@
 
 require 'rbbt/util/simpleopt'
 
-options = SOPT.parse "-h--help:-to--tsv-options*:-p--persistence"
+options = SOPT.get "-h--help:-to--tsv-options*:-p--persistence"
 
 command = ARGV.shift
 file    = ARGV.shift
@@ -10,5 +10,5 @@ file    = ARGV.shift
 case command
 when 'cat'
   puts TSV.new(file, options["tsv-options"].merge(options["persistence"]))
-when '
+when 
 

data/bin/workflow.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/util/workflow'
+
+options = SOPT.get "-h--help:-t--target*:-n--name*"
+
+raise "No target" unless options[:target]
+
+file    = ARGV.shift
+
+if ARGV.empty?
+  data = STDIN.read
+else
+  data = ARGV.shift
+end
+
+job = options[:name] || "Job"
+
+puts "Runing WorkFlow in #{file} for target #{options[:target]}. Job: #{job}"
+WorkFlow.load file, File.join(options[:target], job), data
+puts 
+puts "WorkFlow done. Please find results in: #{File.join(options[:target], job)}"

data/lib/rbbt/util/open.rb

@@ -101,7 +101,7 @@ module Open
     case 
     when Array === grep
       TmpFile.with_file(grep * "\n", false) do |f|
-        CMD.cmd("grep", "-F" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
+        CMD.cmd("grep", "-E" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
       end
     else
       CMD.cmd("grep '#{grep}' -", :in => stream, :pipe => true)
@@ -201,6 +201,7 @@ module Open
   end
 
   def self.write(file, content)
+    FileUtils.mkdir_p File.dirname(file)
     if String === content
       File.open(file, 'w') do |f| f.write content end
     else

data/lib/rbbt/util/path.rb

@@ -123,7 +123,6 @@ module Path
     key_field, fields = TSV.parse_header(self.open, sep, header_hash).values_at(0, 1).flatten.collect{|f| f.extend TSV::Field; f.namespace = namespace; f}.select{|f| f.namespace == namespace}
   end
 
-
   def filename
     self.to_s
   end 
@@ -138,7 +137,9 @@ module Path
   end
 
   def produce
-    return if File.exists? self
+    return self if File.exists? self
+
+    raise "No especified pkg_data for file #{ self }, cannot produce" if pkg_data.nil?
 
     Log.debug("Trying to produce '#{ self }'")
     file, producer = pkg_module.reclaim self
@@ -146,6 +147,8 @@ module Path
     raise "File #{self} has not been claimed, cannot produce" if file.nil? or producer.nil?
 
     pkg_module.produce(self, producer[:get], producer[:subdir], producer[:sharedir])
+
+    self
   end
 end
 

data/lib/rbbt/util/persistence.rb

@@ -21,7 +21,16 @@ module Persistence
  
   def self.get_persistence_file(file, prefix, options = {})
     name = prefix.to_s << ":" << file.to_s << ":"
-    File.join(CACHEDIR, name.to_s.gsub(/\s/,'_').gsub(/\//,'>') + Digest::MD5.hexdigest([file, options].inspect))
+    o = {}
+    options.each do |k,v|
+      if v.inspect =~ /:0x0/
+        o[k] = v.inspect.sub(/:0x[a-f0-9]+@/,'')
+      else
+        o[k] = v
+      end
+    end
+
+    File.join(CACHEDIR, name.to_s.gsub(/\s/,'_').gsub(/\//,'>') + Digest::MD5.hexdigest([file, o].inspect))
   end
 
   def self.get_filename(file)

data/lib/rbbt/util/rake.rb

@@ -12,9 +12,12 @@ module RakeHelper
         FileUtils.chdir chdir if chdir
 
         Rake::FileTask.module_eval do
-          class << self
-            alias_method :old_define_task, :define_task
+          if not self.respond_to? :old_define_task
+            class << self
+              alias_method :old_define_task, :define_task 
+            end 
           end
+
           def self.define_task(file, *args, &block)
             @@files ||= []
             if Hash === file
@@ -64,7 +67,11 @@ module RakeHelper
         end
       end
 
-      load rakefile
+      if block_given?
+        yield
+      else
+        load rakefile
+      end
 
       task(:default) do |t|
         Rake::FileTask.files.each do |file| Rake::Task[file].invoke end

data/lib/rbbt/util/tmpfile.rb

@@ -35,7 +35,7 @@ module TmpFile
 
     result = yield(tmpfile)
 
-    FileUtils.rm tmpfile if File.exists?(tmpfile) and erase
+    FileUtils.rm_rf tmpfile if File.exists?(tmpfile) and erase
 
     result
   end

data/lib/rbbt/util/tsv/index.rb

@@ -139,14 +139,14 @@ class TSV
     #    both have fields => list of names
     #    not both have fields => nil
 
-#    fields2add = case
-#                 when (fields2add.nil? and (other.fields.nil? or self.fields.nil?))
-#                   nil
-#                 when fields2add.nil?
-#                   other.all_fields
-#                 else
-#                   fields2add
-#                 end
+    #    fields2add = case
+    #                 when (fields2add.nil? and (other.fields.nil? or self.fields.nil?))
+    #                   nil
+    #                 when fields2add.nil?
+    #                   other.all_fields
+    #                 else
+    #                   fields2add
+    #                 end
 
     # Determine common fields
     
@@ -171,6 +171,8 @@ class TSV
                                   match_source = (all_fields & match.all_fields).first
                                   index = match.index :target => other.key_field, :fields => match_source
                                   [match_source, index]
+                                when (String === match and match == key_field)
+                                  [:key, other.index]
                                 when String === match
                                   [match, other.index]
                                 when Array === match
@@ -182,32 +184,42 @@ class TSV
     # through
     new = {}
     each do |key,values|
-      source_keys = match_source == :key ? key : values[match_source]
+      source_keys = match_source == :key ? key : values[match_source_position]
       source_keys = [source_keys] unless Array === source_keys
       other_keys = case
-                  when index.nil?
-                    source_keys
-                  else
-                    index.values_at(*source_keys).flatten.compact
-                  end
+                   when index.nil?
+                     source_keys
+                   else
+                     index.values_at(*source_keys).flatten.compact
+                   end
+
       other_keys = other_keys.collect do |other_key| match_index[other_key] end.flatten unless match_index.nil?
 
+
       other_values = other_keys.collect do |other_key|
         next unless other.include? other_key
         new_fields.collect do |field|
           if field == other.key_field
-            other_key
+            if type == :double
+              [other_key]
+            else
+              other_key
+            end
           else
             other[other_key][field]
           end
         end
       end.compact
 
-      if type == :double
-        new_values = values + TSV.zip_fields(other_values)
-      else
-        new_values = values + TSV.zip_fields(other_values).collect{|v| v.first}
-      end
+      other_values = case
+                     when type == :double
+                       TSV.zip_fields(other_values).collect{|v| v.flatten.uniq}
+                     else
+                       TSV.zip_fields(other_values).collect{|v| v.flatten.first}
+                     end
+      
+      new_values = values + other_values
+
       new[key] = new_values
     end
 
@@ -232,7 +244,7 @@ class TSV
       field_values[field] = []
     }
 
-    if type == :double
+    if tsv.type == :double
       tsv.through do |key,entry_values|
         fields.zip(entry_values).each do |field,entry_field_values|
           field_values[field].concat entry_field_values

data/lib/rbbt/util/tsv/parse.rb

@@ -86,7 +86,7 @@ class TSV
 
     if key_field.nil?
       key_pos      = key
-      key_field, fields = nil
+      other_pos    = fields
     else
       all_fields = [key_field].concat other_fields
 
@@ -158,7 +158,7 @@ class TSV
 
         next if data.include?(id) and type != :flat
 
-        if key_field.nil?
+        if other_pos.nil? or (fields == nil and type == :flat)
           other_pos    = (0..(parts.length - 1)).to_a
           other_pos.delete key_pos
         end
@@ -199,7 +199,7 @@ class TSV
         id = ids.shift
         ids.each do |id2| data[id2] = "__Ref:#{id}"  end
 
-        if key_field.nil?
+        if other_pos.nil? or (fields == nil and type == :flat)
           other_pos    = (0..(parts.length - 1)).to_a
           other_pos.delete key_pos
         end
@@ -252,6 +252,8 @@ class TSV
       end
     end
 
+    fields = nil if Fixnum === fields or (Array === fields and fields.select{|f| Fixnum === f}.any?)
+    fields ||= other_fields
     [data, {:key_field => key_field, :fields => fields, :type => type, :case_insensitive => case_insensitive, :namespace => namespace, :datadir => options[:datadir], :identifiers => options[:identifiers], :cast => !!cast}]
   end
 

data/lib/rbbt/util/tsv.rb

@@ -102,7 +102,7 @@ class TSV
         @data = file.data
       when Persistence::TSV === file
         @data = file
-        %w(case_insensitive namespace datadir fields key_field type filename cast).each do |key|
+        %w(case_insensitive namespace identifiers datadir fields key_field type filename cast).each do |key|
           if @data.respond_to?(key.to_sym)  and self.respond_to?("#{key}=".to_sym)
             self.send "#{key}=".to_sym, @data.send(key.to_sym) 
           end
@@ -145,7 +145,7 @@ class TSV
     end
 
     if not extra.nil?
-      %w(case_insensitive namespace datadir fields key_field type filename cast).each do |key| 
+      %w(case_insensitive namespace identifiers datadir fields key_field type filename cast).each do |key| 
         if extra.include? key.to_sym
           self.send("#{key}=".to_sym, extra[key.to_sym])
           if @data.respond_to? "#{key}=".to_sym

data/lib/rbbt/util/workflow.rb

@@ -1,3 +1,90 @@
+require 'rake'
+require 'rbbt/util/rake'
+
 module WorkFlow
+  module Runner
+    def data
+      $_workflow_prereq
+    end
+
+    def input
+      $_workflow_input
+    end
+
+    $_workflow_default_persistence = :string
+    def default_persistence
+      $_workflow_default_persistence
+    end
+
+    def default_persistence=(type)
+      $_workflow_default_persistence = type
+    end
+
+    def step(step_name, options = nil)
+      dependencies, options = case
+                              when ((String === options or Symbol === options) and %w(string marshal tsv tsv_string).include? options.to_s)
+                                [nil, {:persistence_type => options}]
+                              when Hash === options
+                                [nil, options]
+                              else
+                                [options, {}]
+                              end
+
+      options = Misc.add_defaults options, :persistence_type => default_persistence
+      persistence_type = Misc.process_options options, :persistence_type
+      dependencies = Misc.process_options options, :dependencies if options.include? :dependencies
+
+      re = Regexp.new(/(?:^|\/)#{Regexp.quote step_name.to_s}\/.*$/)
 
+      @last_step             = nil unless defined? @last_step
+      @last_persistence_type = nil unless defined? @last_persistence_type
+
+      if dependencies.nil? && ! @last_step.nil?
+        dependencies = @last_step
+      end
+      @last_step = step_name
+
+      # Generate the Hash definition
+      rule_def = case 
+                 when dependencies.nil?
+                   re
+                 when String === dependencies || Symbol === dependencies
+                   {re => lambda{|filename| filename.sub(step_name.to_s, dependencies.to_s) }}
+                 when Array === dependencies
+                   {re => lambda{|filename| dependencies.collect{|dep| filename.sub(step_name.to_s, dep.to_s) } }}
+                 when Proc === dependencies
+                   {re => dependencies}
+                 end
+
+      @last_step = step_name
+      last_persistence_type, @last_persistence_type = @last_persistence_type, persistence_type
+
+      rule rule_def do |t|
+        Persistence.persist(t.name, "", persistence_type, :persistence_file => t.name) do
+          $_workflow_prereq = case
+                   when (t.prerequisites.nil? or (Array === t.prerequisites and t.prerequisites.empty?))
+                     nil
+                   else
+                     Persistence.persist(t.prerequisites.first, "", last_persistence_type, :persistence_file => t.prerequisites.first) do
+                       raise "Error, this file should be produced already"
+                     end
+                   end
+          yield
+        end
+      end
+    end
+  end
+
+  def self.run(file = :default, workflow_input = nil, &block)
+    $_workflow_input = workflow_input
+    RakeHelper.run("Runtime", file) do
+      yield
+    end
+  end
+
+  def self.load(wf_file, file = :default, workflow_input = nil)
+    $_workflow_input = workflow_input
+    RakeHelper.run(wf_file, file)
+  end
 end
+

data/test/rbbt/util/test_R.rb

@@ -11,7 +11,7 @@ class TestR < Test::Unit::TestCase
     tsv2 = tsv.R <<-EOF
 data = data + 1
     EOF
-    puts tsv2.to_s
+    assert_equal "2", tsv2["a"].first
   end
 end
 

data/test/rbbt/util/test_pkg_data.rb

@@ -13,11 +13,6 @@ class TestPKGData < Test::Unit::TestCase
     end
   end
 
-  def test_path
-    assert_equal File.join(Rbbt.datadir, 'Organism/Hsa'), Rbbt.files.Organism.Hsa
-    Rbbt.files.Organism.Hsa.identifiers.produce
-  end
-
   def test_claim_proc
     begin
       assert_nil Rbbt.reclaim(Rbbt.files.foo)

data/test/rbbt/util/test_workflow.rb

@@ -6,9 +6,73 @@ require 'rbbt/util/workflow'
 
 class TestWorkflow < Test::Unit::TestCase
 
-  def test_true
-    assert true
+  def workflow
+    TmpFile.with_file do |dir|
+      old_pdw = FileUtils.pwd
+      begin
+        FileUtils.mkdir dir
+        cd dir
+        yield dir
+      ensure
+        cd old_pdw
+      end
+    end
   end
 
+  def _test_run
+    workflow do |dir|
+      WorkFlow.run do
+        file :foo do |t|
+          touch t.name
+        end
+      end
+      assert File.exists? File.join(dir, 'foo')
+    end
+  end
+
+  def _test_step
+    jobid = 'jobid'
+    target_step = 'last'
+    workflow do |dir|
+
+      WorkFlow.run(File.join(target_step, jobid)) do
+        self.extend WorkFlow::Runner
+
+        step :first do 
+          "Test"
+        end
+
+        step :last do 
+          data.reverse
+        end
+      end
+      assert File.exists? File.join(dir, target_step, 'jobid')
+      assert_equal 'tseT', Open.read(File.join(dir, target_step, 'jobid'))
+    end
+  end
+
+  def test_input
+    jobid = 'jobid'
+    target_step = 'last'
+    message = "Message"
+
+    workflow do |dir|
+
+      WorkFlow.run(File.join(target_step, jobid), message) do
+        self.extend WorkFlow::Runner
+
+        step :first, :marshal do 
+          input
+        end
+
+        step :last do 
+          data.reverse
+        end
+      end
+
+      assert File.exists? File.join(dir, target_step, 'jobid')
+      assert_equal message.reverse, Open.read(File.join(dir, target_step, 'jobid'))
+    end
+  end
 end
 

data/test/rbbt/util/tsv/test_attach.rb

@@ -206,9 +206,6 @@ row1    e
 row2    E
     EOF
 
-
-    require 'rbbt/sources/organism'
-
     Rbbt.claim "data", StringIO.new(content1), "Test1"
     Rbbt.claim "data", StringIO.new(content2), "Test2"
     Rbbt.claim "identifiers", StringIO.new(content_index), "Test2"
@@ -218,7 +215,7 @@ row2    E
     tsv1 = Rbbt.files.Test1.data.tsv :double,  :sep => /\s+/
     tsv2 = Rbbt.files.Test2.data.tsv :double,  :sep => /\s+/
 
-    tsv2.identifiers = Rbbt.files.Test2.identifiers
+    tsv2.identifiers = Rbbt.files.Test2.identifiers.produce
 
     tsv1.attach tsv2, "OtherID", :in_namespace => false
 

data/test/rbbt/util/tsv/test_index.rb

@@ -51,7 +51,6 @@ row3    A    a|B    Id4
     TmpFile.with_file(content) do |filename|
       tsv = TSV.new(File.open(filename), :sep => /\s+/, :key => "OtherID", :persistence => true)
       index = tsv.index(:case_insensitive => false, :order => true)
-      ddd index
       assert_equal "Id1", index['a'].first
       assert_equal "Id3", index['A'].first
       assert_equal "OtherID", index.fields.first
@@ -67,14 +66,16 @@ row3    A    a|B    Id4
 
   #{{{ Test Attach
 
-  def ___test_smart_merge_single
+  def test_smart_merge_single
     content1 =<<-EOF
+#: :case_insensitive=false
 #Id    ValueA    ValueB
 row1    a|aa|aaa    b
 row2    A    B
     EOF
 
     content2 =<<-EOF
+#: :case_insensitive=false
 #ValueC    ValueB    OtherID
 c|cc|ccc    b    Id1|Id2
 C    B    Id3
@@ -91,9 +92,10 @@ C    B    Id3
 
     tsv1 = tsv1.smart_merge tsv2, "ValueB"
 
-    assert_equal "C", tsv1["row2"]["ValueC"]
-    assert %w(c cc ccc).include? tsv1["row1"]["ValueC"]
-    assert_equal "Id1", tsv1["row1"]["OtherID"]
+    assert_equal "C", tsv1["row2"]["ValueC"].first
+    assert %w(c cc ccc).include? tsv1["row1"]["ValueC"].first
+    ddd tsv1
+    assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"].sort
   end
 
   def test_index_to_key

data/test/rbbt/util/tsv/test_parse.rb

@@ -49,6 +49,21 @@ row2    A    B
     end
   end
 
+  def test_unven_flat
+    content =<<-EOF
+row1    a    b
+row2    A    B    C
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      data = {}
+      data, extra = TSV.parse(File.open(filename), :type => :flat, :sep => /\s+/)
+      assert data["row2"].include? "C"
+    end
+  end
+
+
+
   def test_options_line
     content =<<-EOF
 #: :sep=/\\s+/#:case_insensitive=true

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: rbbt-util
 version: !ruby/object:Gem::Version 
-  hash: 13
+  hash: 11
   prerelease: 
   segments: 
   - 2
-  - 0
   - 1
-  version: 2.0.1
+  - 0
+  version: 2.1.0
 platform: ruby
 authors: 
 - Miguel Vazquez
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-02-09 00:00:00 +01:00
+date: 2011-02-10 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -93,6 +93,7 @@ email: miguel.vazquez@fdi.ucm.es
 executables: 
 - tsv.rb
 - tchash.rb
+- workflow.rb
 extensions: []
 
 extra_rdoc_files: 
@@ -160,6 +161,7 @@ files:
 - test/test_rbbt.rb
 - bin/tsv.rb
 - bin/tchash.rb
+- bin/workflow.rb
 has_rdoc: true
 homepage: http://github.com/mikisvaz/rbbt-util
 licenses: []