rbbt-util 3.1.0 → 3.2.0

data/bin/rbbt_query.rb

@@ -9,7 +9,7 @@ options = SOPT.get("-i--identifiers*:-f--format*:-o--organism*:-p--persistence:-
 file   = ARGV[0]
 
 if not File.exists? file
-  base, path = file.match(/(.*)?\.(.*)/).values_at 1, 2
+  base, path = file.match(/([^.]*)\.(.*)/).values_at 1, 2
   require 'rbbt/sources/' << base.to_s.downcase
   klass = Misc.string2const base
   file = klass[path].find

data/lib/rbbt/util/cmd.rb

@@ -3,54 +3,74 @@ require 'rbbt/util/log'
 require 'stringio'
 
 module CMD
-  class CMDError < RBBTError; end
 
+  class CMDError < RBBTError; end
   module SmartIO 
-    def self.tie(io, pid = nil, cmd = "",  post = nil)
-      io.instance_eval{
-        @pid  = pid
-        @cmd  = cmd
-        @post = post
-        alias original_close close
-        def close
-          begin
-            self.original_read unless self.closed? or self.eof?
-            Process.waitpid(@pid) if @pid
-          rescue
-          end
-
-          if $? and not $?.success?
-            Log.debug "Raising exception"
-            exception      = CMDError.new "Command [#{@pid}] #{@cmd} failed with error status #{$?.exitstatus}"
-            raise exception
-          end
+    attr_accessor :pid, :cmd, :post, :in, :out, :err
+    def self.tie(io, pid = nil, cmd = "",  post = nil, sin = nil, out = nil, err = nil)
+      io.extend SmartIO
+      io.pid = pid
+      io.cmd = cmd
+      io.in  = sin 
+      io.out  = out 
+      io.err  = err 
+      io.post = post
+
+      io.class.send(:alias_method, :original_close, :close)
+      io.class.send(:alias_method, :original_read, :read)
+      io
+    end
 
-          @post.call if @post
-          original_close
+    def wait_and_status
+      if @pid
+        begin
+          Process.waitpid(@pid)
+        rescue
         end
 
-        def force_close
-          if @pid
-            Log.debug "Forcing close by killing '#{@pid}'"
-            Process.kill("KILL", @pid)
-            Process.waitpid(@pid)
-          end
-          @post.call if @post
+        Log.debug "Process #{ cmd } succeded" if $? and $?.success?
+
+        if $? and not $?.success?
+          Log.debug "Raising exception"
+          exception = CMDError.new "Command [#{@pid}] #{@cmd} failed with error status #{$?.exitstatus}"
           original_close
+          raise exception
         end
- 
-        alias original_read read
-        def read
-          data = Misc.fixutf8(original_read)
-          self.close unless self.closed?
-          data
-        end
+      end
+    end
 
-      }
-      io
+    def close
+      self.original_read unless self.eof?
+
+      wait_and_status
+
+      @post.call if @post
+
+      original_close unless self.closed?
+    end
+
+    def force_close
+      if @pid
+        Log.debug "Forcing close by killing '#{@pid}'"
+        Process.kill("KILL", @pid)
+        Process.waitpid(@pid)
+      end
+
+      @post.call if @post
+
+      original_close
+    end
+
+    def read(*args)
+      data = original_read(*args)
+
+      self.close if self.eof?
+
+      data
     end
 
- end
+  end
+
 
   def self.process_cmd_options(options = {})
     string = ""
@@ -91,79 +111,107 @@ module CMD
       cmd << " " << cmd_options
     end
 
-    sout, serr = IO.pipe, IO.pipe
-
-    case 
-    when (false and (IO === in_content and not StringIO === in_content))
-      sin = [in_content, nil]
-    else 
-      sin = IO.pipe
-    end
+    in_content = StringIO.new in_content if String === in_content
 
+    sout, serr, sin = IO.pipe, IO.pipe, IO.pipe
 
     pid = fork {
       begin
+        sin.last.close
+        sout.first.close
+        serr.first.close
+
+        io = in_content
+        while IO === io
+          if SmartIO === io
+            io.original_close unless io.closed?
+            io.out.close unless io.out.nil? or io.out.closed?
+            io.err.close unless io.err.nil? or io.err.closed?
+            io = io.in
+          else
+            io.close unless io.closed?
+            io = nil
+          end
+        end
 
-        sin.last.close if sin.last
         STDIN.reopen sin.first
         sin.first.close
 
-        serr.first.close
         STDERR.reopen serr.last
         serr.last.close
 
-        sout.first.close
         STDOUT.reopen sout.last
         sout.last.close
 
         STDOUT.sync = STDERR.sync = true
+        
         exec(cmd)
+
+        exit(-1)
       rescue Exception
+        Log.debug("CMDError: #{$!.message}")
+        ddd $!.backtrace
         raise CMDError, $!.message
       end
     }
+
     sin.first.close
     sout.last.close
     serr.last.close
 
+    sin = sin.last
+    sout = sout.first
+    serr = serr.first
+    
 
     Log.debug "CMD: [#{pid}] #{cmd}"
 
-    case 
-    when String === in_content
-      sin.last.write in_content
-      sin.last.close
-    when in_content.respond_to?(:gets)
+    if in_content.respond_to?(:read)
       Thread.new do
-        while not in_content.eof?
-          sin.last.write in_content.gets
+        begin
+          loop do
+            break if in_content.closed?
+            block = in_content.read 1024
+            break if block.nil? or block.empty?
+            sin.write block
+          end
+
+          sin.close unless sin.closed?
+          in_content.close unless in_content.closed?
+        rescue
+          Process.kill "INT", pid
+          raise $!
         end
-        sin.last.close
       end
+    else
+      sin.close
     end
 
     if pipe
       Thread.new do
-        while l = serr.first.gets
-          Log.log l, stderr if Integer === stderr
+        while line = serr.gets
+          Log.log line, stderr if Integer === stderr
         end
-        serr.first.close
+        serr.close
+        Thread.exit
       end
 
-      SmartIO.tie sout.first, pid, cmd, post
-      sout.first
+      SmartIO.tie sout, pid, cmd, post, in_content, sin, serr
 
+      sout
     else
       err = ""
       Thread.new do
-        while l = serr.first.gets
-          err << l if Integer === stderr
+        while not serr.eof?
+          err << serr.gets if Integer === stderr
         end
-        serr.first.close
+        serr.close
+        Thread.exit
       end
 
-      out = StringIO.new sout.first.read
-      SmartIO.tie out, pid, cmd, post
+      out = StringIO.new sout.read
+      sout.close unless sout.closed?
+      SmartIO.tie out, pid, cmd, post, in_content, sin, serr
 
       Process.waitpid pid
 

data/lib/rbbt/util/fix_width_table.rb

@@ -1,17 +1,16 @@
-
 class FixWidthTable
 
   attr_accessor :filename, :file, :value_size, :record_size, :range, :size
   def initialize(filename, value_size = nil, range = nil, update = false)
     @filename = filename
 
-    if update or %(memmory stringio).include?(filename.to_s.downcase) or not File.exists? filename
+    if update or %w(memory stringio).include?(filename.to_s.downcase) or not File.exists? filename
       Log.debug "Writing FixWidthTable at #{ @filename.inspect }"
       @value_size = value_size
       @range = range
       @record_size = @value_size + (@range ? 12 : 4)
 
-      if %(memory stringio).include? filename.to_s.downcase
+      if %w(memory stringio).include? filename.to_s.downcase
         @file = StringIO.new
       else
         FileUtils.rm @filename if File.exists? @filename
@@ -31,6 +30,20 @@ class FixWidthTable
     end
   end
 
+
+  CONNECTIONS = {} unless defined? CONNECTIONS
+  def self.get(filename, value_size = nil, range = nil, update = false)
+    return self.new(filename, value_size, range, update) if filename == :memory
+    case
+    when (!File.exists?(filename) or update)
+      CONNECTIONS[filename] = self.new(filename, value_size, range, update)
+    when (not CONNECTIONS.include?(filename))
+      CONNECTIONS[filename] = self.new(filename, value_size, range, update)
+    end
+
+    CONNECTIONS[filename] 
+  end
+
   def format(pos, value)
     padding = value_size - value.length
     if range
@@ -82,6 +95,7 @@ class FixWidthTable
   end
 
   def read
+    return if @filename == :memory
     @file.close unless @file.closed?
     @file = File.open(@filename, 'r')
   end
@@ -207,6 +221,7 @@ class FixWidthTable
   end
 
   def [](pos)
+    return [] if size == 0
     if range
       get_range(pos)
     else

data/lib/rbbt/util/misc.rb

@@ -33,13 +33,83 @@ end
 module Misc
   class FieldNotFoundError < StandardError;end
 
+  def self.intersect_sorted_arrays(a1, a2)
+    e1, e2 = a1.shift, a2.shift
+    intersect = []
+    while true
+      case
+      when (e1 and e2)
+        case e1 <=> e2
+        when 0
+          intersect << e1
+          e1, e2 = a1.shift, a2.shift
+        when -1
+          e1 = a1.shift
+        when 1
+          e2 = a2.shift
+        end
+      else
+        break
+      end
+    end
+    intersect
+  end
+
+  def self.merge_sorted_arrays(a1, a2)
+    e1, e2 = a1.shift, a2.shift
+    new = []
+    while true
+      case
+      when (e1 and e2)
+        case e1 <=> e2
+        when 0
+          new << e1 
+          e1, e2 = a1.shift, a2.shift
+        when -1
+          new << e1
+          e1 = a1.shift
+        when 1
+          new << e2
+          e2 = a2.shift
+        end
+      when e2
+        new << e2
+        new.concat a2
+        break
+      when e1
+        new << e1
+        new.concat a1
+        break
+      else
+        break
+      end
+    end
+    new
+  end
+
+  def self.digest(text)
+    Digest::MD5.hexdigest(text)
+  end
+
+  def self.add_method(object, method_name, &block)
+    class << object
+      self
+    end.send :define_method, method_name, block
+  end
+
+  def self.redefine_method(object, old_method, new_method_name, &block)
+    metaclass = class << object; self end
+    metaclass.send :alias_method, new_method_name, old_method
+    metaclass.send :define_method, old_method, &block
+  end
+
   def self.filename?(filename)
     String === filename and filename.length < 1024 and filename.index("\n").nil? and File.exists? filename
   end
 
   def self.lock(file, *args)
     FileUtils.mkdir_p File.dirname(File.expand_path(file)) unless File.exists?  File.dirname(File.expand_path(file))
-    lockfile = Lockfile.new file + '.lock'
+    lockfile = Lockfile.new(file + '.lock')
     lockfile.lock do
       yield file, *args
     end
@@ -104,6 +174,24 @@ module Misc
     counts
   end
 
+  def self.benchmark(repeats = 1)
+    require 'benchmark'
+    res = nil
+    begin
+      measure = Benchmark.measure do
+        repeats.times do
+          res = yield
+        end
+      end
+      puts "Benchmark for #{ repeats } repeats"
+      puts measure
+    rescue Exception
+      puts "Benchmark aborted"
+      raise $!
+    end
+    res
+  end
+
   def self.profile
     require 'ruby-prof'
     RubyProf.start
@@ -279,7 +367,8 @@ module Misc
   # Divides the array into +num+ chunks of the same size by placing one
   # element in each chunk iteratively.
   def self.divide(array, num)
-    chunks = [[]] * num
+    chunks = []
+    num.times do chunks << [] end
     array.each_with_index{|e, i|
       c = i % num
       chunks[c] << e
@@ -287,6 +376,15 @@ module Misc
     chunks
   end
 
+
+
+  def self.process_to_hash(list)
+    result = yield list
+    hash = {}
+    list.zip(result).each do |k,v| hash[k] = v end
+    hash
+  end
+
   IUPAC2BASE = {
     "A" => ["A"],
     "C" => ["C"],
@@ -324,9 +422,9 @@ module PDF2Text
     require 'rbbt/util/cmd'
     require 'rbbt/util/tmpfile'
     require 'rbbt/util/open'
-    TmpFile.with_file(Open.read(filename)) do |pdf|
-      CMD.cmd("pdftotext #{pdf} -", :pipe => false, :stderr => true)
-    end
+
+
+    CMD.cmd("pdftotext - -", :in => Open.open(filename, :nocache => true), :pipe => true, :stderr => true)
   end
 end
 
@@ -381,7 +479,7 @@ class NamedArray < Array
   def zip_fields
     return [] if self.empty?
     zipped = self[0].zip(*self[1..-1])
-    zipped = zipped.collect{|v| NamedArray.name(v, fields)} if fields 
+    zipped = zipped.collect{|v| NamedArray.name(v, fields)}
     zipped 
   end
 
@@ -400,6 +498,7 @@ class NamedArray < Array
       "* #{ field }: #{ Array === value ? value * "|" : value }"
     end * "\n"
   end
+
 end
 
 def benchmark(bench = true)
@@ -430,3 +529,4 @@ def profile(prof = true)
     yield
   end
 end
+