rbbt-util 2.1.0 → 3.0.2

data/lib/rbbt/util/tsv/resource.rb

@@ -0,0 +1,74 @@
+require 'rbbt/util/resource'
+
+module Resource
+  module Path
+    def tsv(type = nil, options = {})
+      if options.empty? and Hash === type
+        options, type = type, nil
+      end
+
+      tsv = TSV.new self, type, options
+      tsv.namespace ||= namespace
+      tsv
+    end
+
+    def namespace_or_dirname
+      namespace || File.basename(File.dirname(self))
+    end
+
+    def index(options = {})
+      TSV.index self, options
+    end
+
+    def open(options = {})
+      produce
+      Open.open(self.find, options)
+    end
+
+    def read(options = {})
+      produce
+      Open.read(self.find, options)
+    end
+
+    def fields(sep = nil, header_hash = nil)
+      produce
+      key, fields, options, line = TSV.parse_header(self.open, sep, header_hash)
+      namespace = options[:namespace] if options.include? namespace
+      fields.collect{|f| f.extend TSV::Field; f.namespace = namespace || namespace_or_dirname ;f}
+    end
+
+    def all_fields(sep = nil, header_hash = nil)
+      produce
+      key, fields, options, line = TSV.parse_header(self.open, sep, header_hash)
+      namespace = options[:namespace] if options.include? namespace
+      [key,fields].flatten.collect{|f| f.extend TSV::Field; f.namespace = namespace || namespace_or_dirname ;f}
+    end
+
+    def fields_in_namespace(sep = nil, header_hash = nil)
+      produce
+      TSV.parse_header(self.open, sep, header_hash)[1].collect{|f| f.extend TSV::Field; f.namespace = namespace ;f}.select{|f| f.namespace == namespace}
+    end
+
+    def all_namespace_fields(namespace, sep = /\t/, header_hash = "#")
+      produce
+      key_field, fields = TSV.parse_header(self.open, sep, header_hash).values_at(0, 1).flatten.collect{|f| f.extend TSV::Field; f.namespace = namespace; f}.select{|f| f.namespace == namespace}
+    end
+
+    def identifier_files
+      dir = self.find.sub(self,'')
+      if dir.nil? or dir.empty?
+        path = File.join(File.dirname(self.find), 'identifiers')
+        path.extend Path
+        path.pkg_module = pkg_module
+        if path.exists?
+          [path]
+        else
+          []
+        end
+      else
+        identifier_files = Misc.find_files_back_to(self.find, 'identifiers', dir)
+        return identifier_files.collect{|f| Resource::Path.path(f)}
+      end
+    end
+  end
+end

data/lib/rbbt/util/workflow.rb

@@ -1,90 +1,114 @@
-require 'rake'
-require 'rbbt/util/rake'
-
+require 'rbbt/util/resource'
+require 'rbbt/util/task'
+require 'rbbt/util/persistence'
 module WorkFlow
-  module Runner
-    def data
-      $_workflow_prereq
+  def self.extended(base)
+    class << base
+      attr_accessor :tasks, :jobdir, :dangling_options, :dangling_option_descriptions,
+        :dangling_option_types, :dangling_option_defaults, :dangling_dependencies, :last_task
     end
 
-    def input
-      $_workflow_input
-    end
+    base.extend Resource
+    base.lib_dir = Resource.caller_base_dir if base.class == Object
+    base.tasks = {}
+    base.jobdir = (File.exists?(base.var.find(:lib)) ? base.var.find(:lib) : base.var.find)
+    base.clear_dangling
+  end
 
-    $_workflow_default_persistence = :string
-    def default_persistence
-      $_workflow_default_persistence
+  def local_persist(*args, &block)
+    argsv = *args
+    options = argsv.pop
+    if Hash === options
+      options.merge!(:persistence_dir => cache.find(:lib))
+      argsv.push options
+    else
+      argsv.push options
+      argsv.push({:persistence_dir => cache.find(:lib)})
     end
+    Persistence.persist(*argsv, &block)
+  end
 
-    def default_persistence=(type)
-      $_workflow_default_persistence = type
-    end
+  def clear_dangling
+    @dangling_options = []
+    @dangling_option_descriptions = {}
+    @dangling_option_types = {} 
+    @dangling_option_defaults = {}
+    @dangling_dependencies = nil
+  end
+
+  def task_option(*args)
+    name, description, type, default = *args
+    @dangling_options << name if name
+    @dangling_option_descriptions[name] = description if description
+    @dangling_option_types[name] = type if type
+    @dangling_option_defaults[name] = default if default
+  end
+
+  def task_dependencies(dependencies)
+    dependencies = [dependencies] unless Array === dependencies
+    @dangling_dependencies = dependencies
+  end
+
+  def process_dangling
+    res = [ 
+      @dangling_options, 
+      Hash[*@dangling_options.zip(@dangling_option_descriptions.values_at(*@dangling_options)).flatten],
+      Hash[*@dangling_options.zip(@dangling_option_types.values_at(*@dangling_options)).flatten],
+      Hash[*@dangling_options.zip(@dangling_option_defaults.values_at(*@dangling_options)).flatten],
+      @dangling_dependencies || @last_task,
+    ]
 
-    def step(step_name, options = nil)
-      dependencies, options = case
-                              when ((String === options or Symbol === options) and %w(string marshal tsv tsv_string).include? options.to_s)
-                                [nil, {:persistence_type => options}]
-                              when Hash === options
-                                [nil, options]
-                              else
-                                [options, {}]
-                              end
-
-      options = Misc.add_defaults options, :persistence_type => default_persistence
-      persistence_type = Misc.process_options options, :persistence_type
-      dependencies = Misc.process_options options, :dependencies if options.include? :dependencies
-
-      re = Regexp.new(/(?:^|\/)#{Regexp.quote step_name.to_s}\/.*$/)
-
-      @last_step             = nil unless defined? @last_step
-      @last_persistence_type = nil unless defined? @last_persistence_type
-
-      if dependencies.nil? && ! @last_step.nil?
-        dependencies = @last_step
-      end
-      @last_step = step_name
-
-      # Generate the Hash definition
-      rule_def = case 
-                 when dependencies.nil?
-                   re
-                 when String === dependencies || Symbol === dependencies
-                   {re => lambda{|filename| filename.sub(step_name.to_s, dependencies.to_s) }}
-                 when Array === dependencies
-                   {re => lambda{|filename| dependencies.collect{|dep| filename.sub(step_name.to_s, dep.to_s) } }}
-                 when Proc === dependencies
-                   {re => dependencies}
-                 end
-
-      @last_step = step_name
-      last_persistence_type, @last_persistence_type = @last_persistence_type, persistence_type
-
-      rule rule_def do |t|
-        Persistence.persist(t.name, "", persistence_type, :persistence_file => t.name) do
-          $_workflow_prereq = case
-                   when (t.prerequisites.nil? or (Array === t.prerequisites and t.prerequisites.empty?))
-                     nil
-                   else
-                     Persistence.persist(t.prerequisites.first, "", last_persistence_type, :persistence_file => t.prerequisites.first) do
-                       raise "Error, this file should be produced already"
-                     end
-                   end
-          yield
-        end
-      end
+    clear_dangling
+    res
+  end
+
+  def task(name, &block)
+    if Hash === name
+      persistence = name.values.first
+      name        = name.keys.first
+    else
+      persistence = :marshal
     end
+
+    options, option_descriptions, option_types, option_defaults, dependencies = process_dangling
+    option_descriptions.delete_if do |k,v| v.nil? end
+    option_types.delete_if do |k,v| v.nil? end
+    option_defaults.delete_if do |k,v| v.nil? end
+    task = Task.new name, persistence, options, option_descriptions, option_types, option_defaults, self, dependencies, self, &block
+    tasks[name] = task
+    @last_task = task
   end
 
-  def self.run(file = :default, workflow_input = nil, &block)
-    $_workflow_input = workflow_input
-    RakeHelper.run("Runtime", file) do
-      yield
+  def job(task, jobname, *args)
+    task = tasks[task]
+    raise "Task #{ task } not found" if task.nil?
+
+    all_options, option_descriptions, option_types, option_defaults = task.recursive_options
+
+    non_optional_arguments = all_options.reject{|option| option_defaults.include? option}
+    run_options = nil
+
+    case
+    when args.length == non_optional_arguments.length
+      run_options = Hash[*non_optional_arguments.zip(args).flatten].merge option_defaults
+    when args.length == non_optional_arguments.length + 1
+      optional_args = args.pop
+      run_options = option_defaults.
+        merge(optional_args).
+        merge(Hash[*non_optional_arguments.zip(args).flatten])
+    else
+      raise "Number of non optional arguments (#{non_optional_arguments * ', '}) does not match given (#{args.flatten * ", "})"
     end
+
+    task.job(jobname,  run_options)
   end
 
-  def self.load(wf_file, file = :default, workflow_input = nil)
-    $_workflow_input = workflow_input
-    RakeHelper.run(wf_file, file)
+  def run(*args)
+    job(*args).run
   end
-end
 
+  def load_job(taskname, job_id)
+    tasks[taskname].load(job_id)
+  end
+
+end

data/test/rbbt/util/test_filecache.rb

@@ -1,12 +1,12 @@
 require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
-require 'rbbt/util/filecache'
 require 'rbbt-util'
+require 'rbbt/util/filecache'
 require 'test/unit'
 
 class TestFileCache < Test::Unit::TestCase
 
   def test_path
-    assert_equal(File.join(Rbbt.cachedir,'3','2','1','123.ext'), FileCache.path('123.ext'))
+    assert_equal(File.join(Rbbt.var.cache.filecache.find(:user),'3','2','1','123.ext'), FileCache.path('123.ext'))
   end
 
   def test_add_read

data/test/rbbt/util/test_misc.rb

@@ -71,8 +71,13 @@ This is an example file. Entries are separated by Entry
     hash = {:a => /test/}
     assert_equal({}, Misc.string2hash(Misc.hash2string(hash)))
  
-
-
  end
 
+  def test_merge
+    a = [[1],[2]]
+    a = NamedArray.name a, %w(1 2)
+    a.merge [3,4]
+    assert_equal [1,3], a[0]
+  end
+
 end

data/test/rbbt/util/test_persistence.rb

@@ -10,7 +10,7 @@ class TestPersistence < Test::Unit::TestCase
       assert File.exists? f
       assert_equal string, Open.read(f)
 
-      rm f
+      FileUtils.rm f
     end
   end
 
@@ -22,7 +22,7 @@ class TestPersistence < Test::Unit::TestCase
       assert_equal object, YAML.load(File.open(f))
       assert_equal YAML.dump(object), Open.read(f)
       
-      rm f
+      FileUtils.rm f
     end
   end
 
@@ -34,7 +34,7 @@ class TestPersistence < Test::Unit::TestCase
       assert_equal object, Marshal.load(File.open(f))
       assert_equal Marshal.dump(object), Open.read(f)
       
-      rm f
+      FileUtils.rm f
     end
   end
 
@@ -42,7 +42,7 @@ class TestPersistence < Test::Unit::TestCase
     object = {:a => 1, :b => 2}
     TmpFile.with_file do |f|
       Persistence.persist("token_file", :Test, :tsv_extra, :persistence_file => f) do 
-        [object, {:fields => ["Number"], :key_field => "Letter", :type => :list, :filename => "foo"}]
+        [object, {:fields => ["Number"], :key_field => "Letter", :type => :single, :cast => :to_i, :filename => "foo"}]
       end
 
       assert File.exists? f
@@ -51,7 +51,7 @@ class TestPersistence < Test::Unit::TestCase
       assert_equal 1, new["a"]
       assert_equal "Letter", new.key_field
       
-      rm f
+      FileUtils.rm f
     end
   end
   
@@ -117,5 +117,40 @@ row2    A    B    Id3
     end
   end
 
+  def test_integer
+    content =<<-EOF
+#Id    ValueA
+row1   1
+row2   2 
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.new(filename, :single, :sep => /\s+/, :cast => :to_i, :persistence => true)
+      assert_equal 1, tsv["row1"]
+    end
+  end
+
+  def test_integer
+    content =<<-EOF
+row1   1 2 3 4 5
+row2   2 4 6 8
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.new(filename, :flat, :sep => /\s+/, :cast => :to_i, :persistence => true)
+      assert_equal [1,2,3,4,5], tsv["row1"]
+    end
+  end
+
+  def test_persist_dir
+    string = Persistence.persist("Test", :Test, :string, :persistence_dir => Rbbt.tmp.test.persistence) do
+      "Test"
+    end
+
+    assert Dir.glob(Rbbt.tmp.test.persistence.find + '*').length == 1
+
+    assert_equal "Test", string
+  end
+
 end
 

data/test/rbbt/util/test_resource.rb

@@ -0,0 +1,92 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/util/resource'
+
+
+module Rbbt
+  extend Resource
+
+  tmp.test_string.define_as_string "Test String"
+  tmp.url.define_as_url "http://www.ruby-lang.org/es/"
+  tmp.Rakefile.define_as_string <<-EOF
+file 'foo' do |t|
+   Open.write(t.name, "Test String")
+end
+  EOF
+
+  tmp.work.define_as_rake tmp.Rakefile.find.produce
+
+  tmp.test.install.xclip.define_as_string <<-EOF
+name="xclip:0.12"
+url="http://downloads.sourceforge.net/project/xclip/xclip/0.12/xclip-0.12.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fxclip%2F&ts=1286472387&use_mirror=sunet"
+
+install_src "$name" "$url"
+  EOF
+
+  FileUtils.chmod 0770, tmp.test.install.xclip.produce
+
+  software.opt.xclip.define_as_install tmp.test.install.xclip.find
+end
+
+Open.cachedir = Rbbt.tmp.cache.find :user
+
+module Phgx
+  extend Resource
+end
+
+
+class TestResource < Test::Unit::TestCase
+  def test_methods
+    assert Resource.methods.include?("resources")
+    assert ! Resource.methods.include?("pkgdir")
+    assert ! Phgx.methods.include?("resources")
+    assert Phgx.methods.include?("pkgdir")
+
+  end
+  def test_resolve
+    assert_equal File.join(ENV['HOME'], '.rbbt/etc/foo'), Resource.resolve('etc/foo', '', :user)
+    assert_equal File.join(ENV['HOME'], '.phgx/etc/foo'), Resource.resolve('etc/foo', 'phgx', :user)
+
+    assert_equal File.join('/', 'etc/foo'), Resource.resolve('etc/foo', '', :global)
+    assert_equal File.join('/', 'etc/phgx/foo'), Resource.resolve('etc/foo', 'phgx', :global)
+
+    assert_equal File.join('/usr/local', 'etc/foo'), Resource.resolve('etc/foo', '', :local)
+    assert_equal File.join('/usr/local', 'etc/phgx/foo'), Resource.resolve('etc/foo', 'phgx', :local)
+
+    assert_equal File.expand_path(File.join(File.dirname(File.expand_path(__FILE__)), '../../../', 'etc/foo')), Resource.resolve('etc/foo', '', :lib)
+    assert_equal File.expand_path(File.join(File.dirname(File.expand_path(__FILE__)), '../../../', 'etc/foo')), Resource.resolve('etc/foo', 'phgx', :lib)
+
+    assert_equal File.join(ENV['HOME'], '.rbbt/etc/foo'), Resource.resolve('etc/foo', '')
+    assert_equal File.join(ENV['HOME'], '.phgx/etc/foo'), Resource.resolve('etc/foo', 'phgx')
+  end
+
+  def test_base
+    assert_equal Rbbt, Rbbt.base
+  end
+
+  def test_path
+    assert_equal File.join(ENV['HOME'], '.rbbt/etc/foo'), Rbbt.etc.foo.find
+    assert_equal File.join(ENV['HOME'], '.rbbt/etc/foo'), Rbbt.etc.foo.find(:user)
+    assert_equal File.join(ENV['HOME'], '.phgx/etc/foo'), Phgx.etc.foo.find
+    assert_equal File.join(ENV['HOME'], '.phgx/etc/foo'), Phgx.etc.foo.find(:user)
+  end
+
+  def test_libdir
+    assert File.exists? Rbbt.share.lib.R["util.R"].find :lib
+    assert File.exists? Rbbt.share.lib.R["util.R"].find 
+  end
+
+  def test_define
+    begin
+      assert_equal "Test String", Rbbt.tmp.test_string.read
+      assert_equal "Test String", Rbbt.tmp.work.foo.read
+    ensure
+      FileUtils.rm Rbbt.tmp.test_string.find if File.exists? Rbbt.tmp.test_string.find
+      FileUtils.rm Rbbt.tmp.url.find if File.exists? Rbbt.tmp.url.find
+    end
+  end
+
+  def test_install
+    assert File.exists?(Rbbt.software.opt.xclip.produce)
+  end
+end
+