rbbt-util 3.2.0 → 3.2.1

data/bin/run_workflow.rb

@@ -0,0 +1,118 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/util/workflow'
+require 'pp'
+
+def usage(task)
+  puts task.usage
+  exit -1
+end
+
+def SOPT_options(task)
+  sopt_options = []
+  task.option_summary.flatten.each do |info|
+    name = info[:name]
+    short = name.to_s.chars.first
+    boolean = info[:type] == :boolean
+    
+    sopt_options << "-#{short}--#{name}#{boolean ? '' : '*'}"
+  end
+
+  sopt_options * ":"
+end
+
+def fix_options(task, job_options)
+  option_types = task.option_summary.flatten.inject({}){|types, new| types[new[:name]] = new[:type]; types}
+
+  job_options_cleaned = {}
+
+  job_options.each do |name, value|
+    value = case
+            when option_types[name] == :float
+              value.to_f
+            when option_types[name] == :integer
+              value.to_i
+            when option_types[name] == :array
+              value.split(/[,|]/)
+            when option_types[name] == :tsv
+              begin
+                if value == '-'
+                  TSV.new(STDIN).to_s :sort
+                else
+                  TSV.new(value).to_s :sort
+                end
+              rescue
+                value
+              end
+            else
+              value
+            end
+    job_options_cleaned[name] = value
+  end
+
+  job_options_cleaned
+end
+
+options = SOPT.get "-t--task*:-l--log*:-h--help:-n--name:-cl--clean:-rcl-recursive_clean"
+
+# Set log, fork, clean, recursive_clean and help
+Log.severity = options[:log].to_i if options.include? :log
+help = !!options.delete(:help)
+do_fork = !!options.delete(:fork)
+clean = !!options.delete(:clean)
+recursive_clean = !!options.delete(:recursive_clean)
+
+# Get workflow
+workflow = ARGV.first
+WorkFlow.require_workflow workflow
+
+# Set task
+namespace, task = nil, nil
+
+case 
+when (not options[:task])
+  workflow_usage if help
+  task = self.last_task
+  namespace = self
+when (options[:task] =~ /\./)
+  namespace, task = options.delete(:task).split('.')
+  namespace = Misc.string2const(namespace)
+else
+  task_name = options.delete(:task)
+  task = self.tasks[task_name]
+end
+
+usage(task) if help
+
+name = options.delete(:name) || "Default"
+
+# get job args
+sopt_option_string = SOPT_options(task)
+job_options = SOPT.get sopt_option_string
+job_options = fix_options(task, job_options)
+
+#- get job
+job = task.job(name, job_options)
+
+# clean job
+job.clean if clean
+job.recursive_clean if recursive_clean
+
+# run
+if do_fork
+  job.fork
+  while not job.done?
+    puts "#{job.step}: #{job.messages.last}"
+    sleep 2
+  end
+else
+  job.run
+end
+
+#- error
+raise job.messages.last if job.error?
+
+#print
+pp job.load

data/lib/rbbt-util.rb

@@ -5,12 +5,9 @@ require 'rbbt/util/tmpfile'
 require 'rbbt/util/filecache'
 require 'rbbt/util/tsv'
 require 'rbbt/util/persistence'
-require 'rbbt/util/bed'
-require 'rbbt/util/cachehelper'
 require 'rbbt/util/misc'
 
 FileCache.cachedir   = Rbbt.var.cache.filecache.find :user
 Open.cachedir        = Rbbt.var.cache["open-remote"].find :user
 TmpFile.tmpdir       = Rbbt.tmp.find :user
 Persistence.cachedir = Rbbt.var.cache.persistence.find :user
-Bed.cachedir         = Rbbt.var.cache["bed-persistence"].find :user

data/lib/rbbt/util/fix_width_table.rb

@@ -11,6 +11,7 @@ class FixWidthTable
       @record_size = @value_size + (@range ? 12 : 4)
 
       if %w(memory stringio).include? filename.to_s.downcase
+        @filename = :memory
         @file = StringIO.new
       else
         FileUtils.rm @filename if File.exists? @filename
@@ -104,6 +105,12 @@ class FixWidthTable
     @file.close
   end
 
+  def dump
+    read
+    @file.rewind
+    @file.read
+  end
+
   #{{{ Adding data
 
   def add_point(data)

data/lib/rbbt/util/misc.rb

@@ -33,6 +33,16 @@ end
 module Misc
   class FieldNotFoundError < StandardError;end
 
+  def self.in_dir(dir)
+    old_pwd = FileUtils.pwd
+    begin
+      FileUtils.cd dir
+      yield
+    ensure
+      FileUtils.cd old_pwd
+    end
+  end
+
   def self.intersect_sorted_arrays(a1, a2)
     e1, e2 = a1.shift, a2.shift
     intersect = []
@@ -98,6 +108,7 @@ module Misc
   end
 
   def self.redefine_method(object, old_method, new_method_name, &block)
+    return if object.respond_to? new_method_name
     metaclass = class << object; self end
     metaclass.send :alias_method, new_method_name, old_method
     metaclass.send :define_method, old_method, &block
@@ -376,13 +387,16 @@ module Misc
     chunks
   end
 
+  def self.merge2hash(list1, list2)
+    hash = {}
+    list1.zip(list2).each do |k,v| hash[k] = v end
+    hash
+  end
 
 
   def self.process_to_hash(list)
     result = yield list
-    hash = {}
-    list.zip(result).each do |k,v| hash[k] = v end
-    hash
+    merge2hash(list, result)
   end
 
   IUPAC2BASE = {
@@ -424,7 +438,9 @@ module PDF2Text
     require 'rbbt/util/open'
 
 
-    CMD.cmd("pdftotext - -", :in => Open.open(filename, :nocache => true), :pipe => true, :stderr => true)
+    TmpFile.with_file(Open.open(filename, :nocache => true).read) do |pdf_file|
+      CMD.cmd("pdftotext #{pdf_file} -", :pipe => false, :stderr => true)
+    end
   end
 end
 

data/lib/rbbt/util/persistence.rb

@@ -23,6 +23,13 @@ module Persistence
   def self.get_persistence_file(file, prefix, options = {})
     persistence_dir = Misc.process_options options, :persistence_dir
     persistence_dir ||= CACHEDIR
+
+    if options.include? :filters
+      options[:filters].each do |match,value|
+        file = file + "&F[#{match}=#{Misc.digest(value.inspect)}]"
+      end
+    end
+
     name = prefix.to_s.dup << ":" << file.to_s << ":"
 
     options_md5 = Misc.hash2md5 options
@@ -171,22 +178,28 @@ module Persistence
       Log.debug "Creating #{ persistence_file }. Prefix = #{prefix}"
 
       res = yield file, options, filename, persistence_file
+
       serializer = tsv_serializer res
 
-      if File.exists? persistence_file
-        Log.debug "Erasing old #{ persistence_file }. Prefix = #{prefix}"
-        FileUtils.rm persistence_file
-      end
+      if TCHash === res
+        per = res
+      else
 
-      Log.debug "Dump data into '#{persistence_file}'"
-      per = Persistence::TSV.get persistence_file, true, serializer
+        if File.exists? persistence_file
+          Log.debug "Erasing old #{ persistence_file }. Prefix = #{prefix}"
+          FileUtils.rm persistence_file
+        end
 
-      per.write
-      per.merge! res
+        Log.debug "Dump data into '#{persistence_file}'"
+        per = Persistence::TSV.get persistence_file, true, serializer
 
-      Persistence::TSV::FIELD_INFO_ENTRIES.keys.each do |key| 
-        if res.respond_to?(key.to_sym)  and per.respond_to?("#{key}=".to_sym)
-          per.send "#{key}=".to_sym, res.send(key.to_sym) 
+        per.write
+        per.merge! res
+
+        Persistence::TSV::FIELD_INFO_ENTRIES.keys.each do |key| 
+          if res.respond_to?(key.to_sym) and per.respond_to?("#{key}=".to_sym)
+            per.send "#{key}=".to_sym, res.send(key.to_sym) 
+          end
         end
       end
 
@@ -293,27 +306,36 @@ module Persistence
         FileUtils.rm persistence_file
       end
 
-      max_length = res.collect{|k,v| k.length}.max
 
-      if range
-        begin
-          fwt = FixWidthTable.new persistence_file, max_length, true
-          fwt.add_range res
-        rescue
-          FileUtils.rm persistence_file if File.exists? persistence_file
-          raise $!
-        end
+      if FixWidthTable === res and res.filename == :memory 
+        Log.debug "Dumping memory FWT into #{ persistence_file }. Prefix = #{prefix}"
+        FileUtils.mkdir_p File.dirname(persistence_file) unless File.exists? File.dirname(persistence_file)
+        Open.write(persistence_file, res.dump)
+        fwt = FixWidthTable.get persistence_file
       else
-        begin
-          fwt = FixWidthTable.new persistence_file, max_length, false
-          fwt.add_point res
-        rescue
-          FileUtils.rm persistence_file
-          raise $!
+
+        max_length = res.collect{|k,v| k.length}.max
+
+        if range
+          begin
+            fwt = FixWidthTable.get persistence_file, max_length, true
+            fwt.add_range res
+          rescue
+            FileUtils.rm persistence_file if File.exists? persistence_file
+            raise $!
+          end
+        else
+          begin
+            fwt = FixWidthTable.get persistence_file, max_length, false
+            fwt.add_point res
+          rescue
+            FileUtils.rm persistence_file
+            raise $!
+          end
         end
-      end
 
-      fwt.read
+        fwt.read
+      end
 
       fwt
     else
@@ -327,6 +349,7 @@ module Persistence
 
   def self.persist(file, prefix = "", persistence_type = :string, options = {}, &block)
     options = Misc.add_defaults options, :persistence => true
+
     persistence =
       Misc.process_options options, :persistence
 

data/lib/rbbt/util/resource.rb

@@ -29,6 +29,7 @@ module Resource
       return dir if File.exists? File.join(dir, 'lib')
       file = File.dirname file
     end
+
     return nil
   end
 

data/lib/rbbt/util/task.rb

@@ -55,10 +55,20 @@ class Task
     [run_options, args, optional_args]
   end
 
+  def pull_from_hash(args, optional_args)
+    option_summary.first.each do |info|
+      name = info[:name]
+      if optional_args.include? name
+        args.push optional_args.delete name
+      end
+    end
+  end
+
   def setup(jobname, args, optional_args, dependencies)
     previous_jobs = []
     required_files = []
   
+    pull_from_hash(args, optional_args)
     run_options, args, optional_args = process_options args, optional_args
 
     dependencies.each do |dependency|
@@ -89,6 +99,7 @@ class Task
       optional_args = {}
     end
 
+
     previous_jobs, required_files, run_options = setup(jobname, args, optional_args, dependencies)
 
     job_id = self.job_id jobname, run_options, previous_jobs

data/lib/rbbt/util/task/job.rb

@@ -2,7 +2,7 @@ require 'rbbt/util/misc'
 
 class Task
   class Job
-    attr_accessor :task, :id, :name, :options, :previsous_jobs, :required_files, :pid, :path, :previous_jobs, :input
+    attr_accessor :task, :id, :name, :options, :previous_jobs, :required_files, :pid, :path, :previous_jobs, :input
 
     IDSEP = "_"
 
@@ -285,6 +285,8 @@ class Task
         Marshal.load(Open.read(path))
       when :yaml
         YAML.load(Open.read(path))
+      when nil
+        nil
       end
     end
 

data/lib/rbbt/util/tc_hash.rb

@@ -1,5 +1,6 @@
 require 'rbbt/util/misc'
 require 'tokyocabinet'
+require 'set'
 
 class TCHash < TokyoCabinet::HDB
   class OpenError < StandardError;end
@@ -10,6 +11,11 @@ class TCHash < TokyoCabinet::HDB
     def self.load(str); str.unpack("l").first; end
   end
 
+  class FloatSerializer
+    def self.dump(i); [i].pack("d"); end
+    def self.load(str); str.unpack("d").first; end
+  end
+
   class IntegerArraySerializer
     def self.dump(a); a.pack("l*"); end
     def self.load(str); str.unpack("l*"); end
@@ -40,8 +46,29 @@ class TCHash < TokyoCabinet::HDB
     end
   end
 
+  class TSVSerializer
+    def self.dump(tsv)
+      tsv.to_s
+    end
+
+    def self.load(string)
+      TSV.new StringIO.new(string)
+    end
+  end
 
-  ALIAS = {:integer => IntegerSerializer, :integer_array => IntegerArraySerializer, :marshal => Marshal, nil => Marshal, :single => StringSerializer, :list => StringArraySerializer, :double => StringDoubleArraySerializer}
+
+
+  ALIAS = {
+    :integer => IntegerSerializer, 
+    :float => FloatSerializer, 
+    :integer_array => IntegerArraySerializer,
+    :marshal => Marshal,
+    :single => StringSerializer,
+    :string => StringSerializer,
+    :list => StringArraySerializer,
+    :double => StringDoubleArraySerializer,
+    :tsv => TSVSerializer
+  }
 
   CONNECTIONS = {}
 
@@ -49,7 +76,6 @@ class TCHash < TokyoCabinet::HDB
     :type             => '__tokyocabinet_hash_type',
     :serializer       => '__tokyocabinet_hash_serializer',
     :identifiers      => '__tokyocabinet_hash_identifiers',
-    :datadir          => '__tokyocabinet_hash_datadir',
     :fields           => '__tokyocabinet_hash_fields',
     :key_field        => '__tokyocabinet_hash_key_field',
     :filename         => '__tokyocabinet_hash_filename',
@@ -74,7 +100,13 @@ class TCHash < TokyoCabinet::HDB
     out(key) or raise "Not deleted"
   end
 
-  attr_accessor :serializer
+  alias original_include? include?
+  def include?(key)
+    return nil unless String === key
+    original_include? key
+  end
+
+  attr_accessor :serializer, :path_to_db
   def serializer=(serializer)
     
     if ALIAS.include? serializer.to_sym
@@ -97,17 +129,20 @@ class TCHash < TokyoCabinet::HDB
 
     @write = write
 
-    if self.include? FIELD_INFO_ENTRIES[:serializer]
-      serializer_str = self.original_get_brackets(FIELD_INFO_ENTRIES[:serializer])
+    if @serializer.nil?
 
-      mod = Misc.string2const serializer_str
-      @serializer = mod
+      if self.include? FIELD_INFO_ENTRIES[:serializer]
+        serializer_str = self.original_get_brackets(FIELD_INFO_ENTRIES[:serializer])
 
-    else
-      raise "No serializer specified" if serializer.nil?
+        mod = Misc.string2const serializer_str
+        @serializer = mod
 
-      self.original_set_brackets(FIELD_INFO_ENTRIES[:serializer], serializer.to_s) unless self.include? FIELD_INFO_ENTRIES[:serializer]
-      @serializer = serializer
+      else
+        raise "No serializer specified" if (serializer || @serializer).nil?
+
+        self.original_set_brackets(FIELD_INFO_ENTRIES[:serializer], serializer.to_s) unless self.include? FIELD_INFO_ENTRIES[:serializer]
+        @serializer = serializer
+      end
     end
   end
 
@@ -116,31 +151,44 @@ class TCHash < TokyoCabinet::HDB
   end
 
   def write
+    self.sync
     self.close
     self.open(true)
   end
 
   def read
+    self.sync
     self.close
     self.open(false)
   end
 
-  def initialize(path, write = false, serializer = Marshal)
+  def initialize(path, write = false, serializer = nil)
     super()
 
-    serializer = ALIAS[serializer] if ALIAS.include? serializer
+    if ALIAS.include? serializer
+      @serializer = ALIAS[serializer]
+    else
+      @serializer = serializer
+    end
 
     @path_to_db = path
 
     if write || ! File.exists?(@path_to_db)
+      @serializer = Marshal if @serializer.nil?
       self.setcache(100000) or raise "Error setting cache"
-      self.open(true, serializer)
+      self.open(true, @serializer)
+      self.original_set_brackets(FIELD_INFO_ENTRIES[:serializer], @serializer.to_s)
     else
       self.open(false)
     end
   end
 
-  def self.get(path, write = false, serializer = Marshal)
+  def self.get(path, write = false, serializer = nil)
+    if not (TrueClass === write or FalseClass === write) and serializer.nil?
+      serializer = write
+      write = false
+    end
+
     if ALIAS.include? serializer
       serializer = ALIAS[serializer] 
     else
@@ -213,6 +261,7 @@ class TCHash < TokyoCabinet::HDB
     values.collect{|v| serializer.load(v)}
   end
 
+  alias real_original_each each
   # This version of each fixes a problem in ruby 1.9. It also
   # removes the special entries
   def each(&block)
@@ -224,17 +273,26 @@ class TCHash < TokyoCabinet::HDB
     keys.zip(values.collect{|v| serializer.load v}).each &block
   end
 
+  def each(&block)
+    skippable = Set.new(FIELD_INFO_ENTRIES.values)
+    real_original_each do |key, value|
+      block.call(key, serializer.load(value)) unless skippable.include? key
+    end
+  end
+
   alias original_each each
   
-  def collect
+  def collect(&block)
+    skippable = Set.new(FIELD_INFO_ENTRIES.values)
     res = []
-    self.each{|k, v| 
+    real_original_each do |key,value|
+      next if skippable.include? key
       if block_given?
-        res << yield(k,v)
+        block.call(key, serializer.load(value)) 
       else
-        res << [k,v]
+        res << [key, value]
       end
-    }
+    end
     res
   end
 
@@ -251,5 +309,16 @@ class TCHash < TokyoCabinet::HDB
       raise "Transaction cannot initiate"
     end
   end
+  
+  def clear
+    special_values = FIELD_INFO_ENTRIES.values.sort.collect{|k|  self.original_get_brackets(k)}
+    restore = ! write?
+    write if restore
+    vanish
+    FIELD_INFO_ENTRIES.values.sort.zip(special_values).each{|k,v|
+      self.original_set_brackets(k,v) unless v.nil?
+    }
+    read if restore
+  end
 
 end