rbbt-util 5.2.4 → 5.3.0

data/lib/rbbt/resource.rb

@@ -4,11 +4,7 @@ require 'rbbt/util/chain_methods'
 require 'rbbt/resource/path'
  
 module Resource
-  extend ChainMethods
-  self.chain_prefix = :resource
-
   def self.extended(base)
-    setup_chains(base)
     if not base.respond_to? :pkgdir
       class << base
         attr_accessor :pkgdir, :subdir, :resources, :rake_dirs
@@ -26,8 +22,7 @@ module Resource
     Path.setup @subdir || "", @pkgdir, self
   end
 
-  def resource_method_missing(name, prev = nil, *args)
-    # Fix problem with ruby 1.9 calling methods by its own initiative. ARG
+  def method_missing(name, prev = nil, *args)
     if prev.nil?
       root.send(name, *args)
     else

data/lib/rbbt/resource/path.rb

@@ -1,29 +1,15 @@
-require 'rbbt/util/chain_methods'
 require 'rbbt/resource/util'
 require 'rbbt/tsv'
 
 module Path
   attr_accessor :resource, :pkgdir
 
-  extend ChainMethods
-  self.chain_prefix = :path
-
   def self.setup(string, pkgdir = nil, resource = nil)
     string.extend Path
     string.pkgdir = pkgdir || 'rbbt'
     string.resource = resource
     string
   end
-
-  def self.extended(string)
-    setup_chains(string)
-    if not string.respond_to? :byte
-      class << string
-        alias byte path_clean_get_brackets
-      end
-    end
-  end
-
   def join(name)
     if self.empty?
       Path.setup name.to_s, @pkgdir, @resource
@@ -37,24 +23,29 @@ module Path
   end
 
   def glob(pattern = '*')
-    Dir.glob(File.join(self, pattern)).collect{|f| Path.setup(f, self.resource, self.pkgdir)}
+    Dir.glob(File.join(Regexp.quote(self), pattern)).collect{|f| Path.setup(f, self.resource, self.pkgdir)}
   end
 
-  def path_get_brackets(name)
+  def [](name, orig = false)
+    return super(name) if orig
     join name
   end
 
-  def path_method_missing(name, prev = nil, *args, &block)
+  def byte(pos)
+    send(:[], pos, true)
+  end
+
+  def method_missing(name, prev = nil, *args, &block)
     if block_given?
-      path_clean_method_missing name, prev, *args, &block
+      super name, prev, *args, &block
     else
       # Fix problem with ruby 1.9 calling methods by its own initiative. ARG
-      path_clean_method_missing(name, prev, *args) if name.to_s =~ /^to_/
-        if prev.nil?
-          join name
-        else
-          join(prev).join(name)
-        end
+      super(name, prev, *args) if name.to_s =~ /^to_/
+      if prev.nil?
+        join name
+      else
+        join(prev).join(name)
+      end
     end
   end
 
@@ -135,7 +126,8 @@ module Path
 
   def produce(force = false)
     path = self.find
-    return self if File.exists?(path) and not force
+
+    return self if Open.exists?(path.to_s) and not force
 
     raise "No resource defined to produce file: #{ self }" if resource.nil?
 
@@ -153,8 +145,8 @@ module Path
   end
 
 
-  def open(options = {})
-    Open.open(self.produce.find, options)
+  def open(options = {}, &block)
+    Open.open(self.produce.find, options, &block)
   end
 
   def to_s
@@ -181,6 +173,10 @@ module Path
     YAML.load self.open
   end
 
+  def pipe_to(cmd, options = {})
+    CMD.cmd(cmd, {:in => self.open, :pipe => true}.merge(options))
+  end
+
   def index(options = {})
     TSV.index(self.produce, options)
   end

data/lib/rbbt/tsv.rb

@@ -16,6 +16,14 @@ require 'rbbt/tsv/attach'
 require 'rbbt/tsv/filter'
 
 module TSV
+  class << self
+    attr_accessor :lock_dir
+    
+    def lock_dir
+      @lock_dir ||= Rbbt.tmp.tsv_open_locks.find
+    end
+  end
+
   def self.setup(hash, options = {})
     options = Misc.add_defaults options, :default_value => []
     default_value = Misc.process_options options, :default_value
@@ -48,7 +56,7 @@ module TSV
 
     data = nil
 
-    lock_filename = filename.nil? ? nil : Persist.persistence_path(filename, {:dir => Rbbt.tmp.tsv_open_locks.find})
+    lock_filename = filename.nil? ? nil : Persist.persistence_path(filename, {:dir => TSV.lock_dir})
     Misc.lock lock_filename  do
       data = Persist.persist_tsv source, filename, options, persist_options do |data|
         if serializer
@@ -63,7 +71,7 @@ module TSV
 
         data.filename = filename.to_s unless filename.nil?
         if data.identifiers.nil? and Path === filename and filename.identifier_file_path
-          data.identifiers = filename.identifier_file_path.to_s 
+          data.identifiers = filename.identifier_file_path.to_s
         end
 
         data
@@ -74,11 +82,26 @@ module TSV
 
     data.entity_options = entity_options
 
+    if Path === source and data.identifiers
+      data.identifiers = Path.setup(data.identifiers, source.pkgdir, source.resource)
+    end
+
     data
   end
 
   def self.parse_header(stream, options = {})
-    Parser.new stream, options
+    case
+    when Path === stream 
+      stream.open do |f|
+        Parser.new f, options
+      end
+    when (String === stream and stream.length < 300 and Open.exists? stream)
+      stream.open do |f|
+        Parser.new f, options
+      end
+    else
+      Parser.new stream, options
+    end
   end
 
   def self.parse(stream, data, options = {})
@@ -96,11 +119,17 @@ module TSV
 
     if TokyoCabinet::HDB === data and parser.straight
       data.close
+      pos = stream.pos if stream.respond_to? :pos
       begin
-        CMD.cmd("tchmgr importtsv '#{data.persistence_path}'", :in => stream, :log => false)
+        CMD.cmd("tchmgr importtsv '#{data.persistence_path}'", :in => stream, :log => false, :dont_close_in => true)
       rescue
         Log.debug("tchmgr importtsv failed for: #{data.persistence_path}")
         Log.debug($!.message)
+        if stream.respond_to? :seek 
+          stream.seek pos
+        else
+          #raise "tchmgr import failed and cannot restore stream"
+        end
       end
       data.write
     end

data/lib/rbbt/tsv/accessor.rb

@@ -1,13 +1,18 @@
-require 'rbbt/util/chain_methods'
+#require 'rbbt/util/chain_methods'
 require 'yaml'
 module TSV
-  extend ChainMethods
-  self.chain_prefix = :tsv
+  #extend ChainMethods
+  #self.chain_prefix = :tsv
 
-  NIL_YAML = "--- \n"
+  TSV_SERIALIZER = YAML
+  SERIALIZED_NIL = TSV_SERIALIZER.dump nil
 
   attr_accessor :unnamed, :serializer_module, :entity_options, :entity_templates
 
+  def annotate(tsv)
+    TSV.setup(tsv, :key_field => key_field, :fields => fields, :namespace => namespace, :entity_options => entity_options, :type => type, :filename => filename, :identifiers => identifiers)
+  end
+
   def entity_options
     if @entity_options.nil?
       @entity_options = namespace ? {:namespace => namespace, :organism => namespace} : {}
@@ -16,6 +21,12 @@ module TSV
     @entity_options
   end
 
+  def entity_options=(options)
+    @entity_options = options
+    @entity_templates = nil
+  end
+
+
   def entity_templates
     @entity_templates ||= {}
   end
@@ -70,7 +81,7 @@ module TSV
   end
 
   def self.extended(data)
-    setup_chains(data)
+    #setup_chains(data)
 
     if not data.respond_to? :write
       class << data
@@ -96,10 +107,10 @@ module TSV
     end
 
     if not data.respond_to? :serialized_get
-      class << data
-        alias serialized_get tsv_clean_get_brackets
-        alias serialized_set tsv_clean_set_brackets
-      end
+      #class << data
+      #  alias serialized_get []
+      #  alias serialized_set []=
+      #end
     end
   end
 
@@ -109,13 +120,13 @@ module TSV
   ENTRY_KEYS = []
 
   #{{{ Chained Methods
-  def tsv_empty?
+  def empty?
     length == 0
   end
 
-  def tsv_get_brackets(key)
-    value = serialized_get(key)
-    return value if value.nil? or @unnamed or fields.nil?
+  def [](key, clean = false)
+    value = (self.respond_to?(:serialized_get) and not clean) ? serialized_get(key) : super(key)
+    return value if value.nil? or @unnamed or clean == :entry_key or fields.nil?
 
     case type
     when :double, :list
@@ -128,41 +139,44 @@ module TSV
     value
   end
 
-  def tsv_set_brackets(key,value)
+  def []=(key, value, clean = false)
+    return super(key, value) if clean or not self.respond_to?(:serialized_set)
     serialized_set(key, value)
   end
 
-  def tsv_keys
-    keys = tsv_clean_keys - ENTRY_KEYS
+  def keys
+    keys = super - ENTRY_KEYS
     return keys if @unnamed or key_field.nil?
 
     prepare_entity(keys, key_field, entity_options.merge(:dup_array => true))
   end
 
-  def tsv_values
+  def values
     values = chunked_values_at(keys)
     return values if @unnamed or fields.nil?
 
     case type
     when :double, :list
       values.each{|value| setup_array value, fields, nil, entity_options}
-    when :flat, :single
+    when :single
+      values = prepare_entity(values, fields.first, entity_options)
+    when :flat
       values = values.collect{|v| prepare_entity(v, fields.first, entity_options)}
     end
       
     values
   end
 
-  def tsv_each
+  def each
     fields = self.fields
 
     serializer = self.serializer
     serializer_module = SERIALIZER_ALIAS[serializer] unless serializer.nil?
-    tsv_clean_each do |key, value|
+    super do |key, value|
       next if ENTRY_KEYS.include? key
 
       # TODO Update this to be more efficient
-      value = serializer_module.load(value) unless serializer.nil?
+      value = serializer_module.load(value) unless serializer.nil? or FalseClass === serializer
 
       # Annotated with Entity and NamedArray
       if not @unnamed
@@ -182,10 +196,10 @@ module TSV
     end
   end
 
-  def tsv_collect
+  def collect
     serializer = self.serializer
     serializer_module = SERIALIZER_ALIAS[serializer] unless serializer.nil?
-    tsv_clean_collect do |key, value|
+    super do |key, value|
       next if ENTRY_KEYS.include? key
 
       # TODO Update this to be more efficient
@@ -212,15 +226,15 @@ module TSV
     end
   end
 
-  def tsv_size
+  def size
     keys.length
   end
 
-  def tsv_length
+  def length
     keys.length
   end
 
-  def tsv_values_at(*keys)
+  def values_at(*keys)
     keys.collect do |key|
       self[key]
     end
@@ -236,7 +250,7 @@ module TSV
 
   #{{{ Sorting
 
-  def tsv_sort_by(field = nil, just_keys = false, &block)
+  def sort_by(field = nil, just_keys = false, &block)
     field = :all if field.nil?
     if field == :all
       elems = collect
@@ -249,7 +263,7 @@ module TSV
         end
       when :list, :flat
         through :key, field do |key, fields|
-          elems << [key, fields.first]
+          elems << [key, fields]
         end
       when :double
         through :key, field do |key, fields|
@@ -267,12 +281,33 @@ module TSV
           elems.sort_by{|key, value| key }
         end
       else
+        sorted = elems.sort do |a, b| 
+          a_value = a.last
+          b_value = b.last
+          case
+          when ((a_value.nil? or (a_value.respond_to?(:empty?) and a_value.empty?)) and (b_value.nil? or (b_value.respond_to?(:empty?) and b_value.empty?)))
+            0
+          when (a_value.nil? or (a_value.respond_to?(:empty?) and a_value.empty?))
+            -1
+          when (b_value.nil? or (b_value.respond_to?(:empty?) and b_value.empty?))
+            1
+          when Array === a_value
+            if a_value.length == 1 and b_value.length == 1
+              a_value.first <=> b_value.first
+            else
+              a_value.length <=> b_value.length
+            end
+          else
+            a_value <=> b_value
+          end
+        end
         if just_keys
-          keys = elems.sort_by{|key, value| value }.collect{|key, value| key}
+          #keys = elems.sort_by{|key, value| value }.collect{|key, value| key}
+          keys = sorted.collect{|key, value| key}
           keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true))
           keys
         else
-          elems.sort_by{|key, value| value }.collect{|key, value| [key, self[key]]}
+          sorted.collect{|key, value| [key, self[key]]}
         end
       end
     else
@@ -310,12 +345,13 @@ module TSV
     entries.each do |entry|
       key = KEY_PREFIX + entry
       ENTRY_KEYS << key
-      self.module_eval "
+      line = __LINE__; self.module_eval "
 attr_accessor :#{entry}
 
 def #{ entry }
   if not defined? @#{entry}
-    @#{entry} = (value = self.tsv_clean_get_brackets('#{key}')).nil? ? nil : YAML.load(value)
+    # @#{entry} = (value = self.clean_get_brackets('#{key}')).nil? ? nil : TSV_SERIALIZER.load(value)
+    @#{entry} = (value = self.send(:[], '#{key}', :entry_key)).nil? ? nil : TSV_SERIALIZER.load(value)
   end
   @#{entry}
 end
@@ -325,33 +361,36 @@ if '#{entry}' == 'serializer'
 
   def #{ entry }=(value)
     @#{entry} = value
-    self.tsv_clean_set_brackets '#{key}', value.nil? ? NIL_YAML : value.to_yaml
+    #self.tsv_clean_set_brackets '#{key}', value.nil? ? SERIALIZED_NIL : value.to_yaml
+    self.send(:[]=, '#{key}', value.nil? ? SERIALIZED_NIL : value.to_yaml, true)
 
     return if value.nil?
 
     self.serializer_module = SERIALIZER_ALIAS[value.to_sym]
 
     if serializer_module.nil?
-      class << self
-        alias serialized_get tsv_clean_get_brackets
-        alias serialized_set tsv_clean_set_brackets
-      end
+      #class << self
+      #  alias serialized_get tsv_clean_get_brackets
+      #  alias serialized_set tsv_clean_set_brackets
+      #end
 
     else
       class << self
 
         define_method :serialized_get do |key|
           return nil unless self.include? key
-          res = tsv_clean_get_brackets(key)
+          res = self.send(:[], key, true)
           return res if res.nil?
           self.serializer_module.load(res)
         end
 
         define_method :serialized_set do |key, value|
           if value.nil?
-            tsv_clean_set_brackets key, value
+            self.send(:[]=, key, value, true)
+            #tsv_clean_set_brackets key, value
           else
-            tsv_clean_set_brackets key, self.serializer_module.dump(value)
+            self.send(:[]=, key, self.serializer_module.dump(value), true)
+            #tsv_clean_set_brackets key, self.serializer_module.dump(value)
           end
         end
       end
@@ -361,10 +400,11 @@ if '#{entry}' == 'serializer'
 else
   def #{ entry }=(value)
     @#{entry} = value
-    self.tsv_clean_set_brackets '#{key}', value.nil? ? NIL_YAML : value.to_yaml
+    self.send(:[]=, '#{key}', value.nil? ? SERIALIZED_NIL : value.to_yaml, true)
+    #self.tsv_clean_set_brackets '#{key}', value.nil? ? SERIALIZED_NIL : value.to_yaml
   end
 end
-  "
+  ", __FILE__, line
     end
   end
 
@@ -378,8 +418,9 @@ end
     :serializer
 
   def fields
-    @fields ||= YAML.load(self.tsv_clean_get_brackets("__tsv_hash_fields") || "--- \n")
-    if @fields.nil? or @unnamed
+    #@fields ||= TSV_SERIALIZER.load(self.tsv_clean_get_brackets("__tsv_hash_fields") || SERIALIZED_NIL)
+    @fields ||= TSV_SERIALIZER.load(self.send(:[], "__tsv_hash_fields", :entry_key) || SERIALIZED_NIL)
+    if true or @fields.nil? or @unnamed
       @fields
     else
       @named_fields ||= NamedArray.setup @fields, @fields, nil, entity_options, entity_templates
@@ -387,13 +428,15 @@ end
   end
 
   def namespace=(value)
-    self.tsv_clean_set_brackets "__tsv_hash_namespace", value.nil? ? NIL_YAML : value.to_yaml
+    #self.tsv_clean_set_brackets "__tsv_hash_namespace", value.nil? ? SERIALIZED_NIL : value.to_yaml
+    self.send(:[]=, "__tsv_hash_namespace", value.nil? ? SERIALIZED_NIL : value.to_yaml, true)
     @namespace = value
     @entity_options = nil
   end
 
   def fields=(value)
-    self.tsv_clean_set_brackets "__tsv_hash_fields", value.nil? ? NIL_YAML : value.to_yaml
+    #self.tsv_clean_set_brackets "__tsv_hash_fields", value.nil? ? SERIALIZED_NIL : value.to_yaml
+    self.send(:[]=, "__tsv_hash_fields", value.nil? ? SERIALIZED_NIL : value.to_yaml, true)
     @fields = value
     @named_fields = nil
   end
@@ -422,7 +465,7 @@ end
     when Path === filename
       filename.identifier_files
     when filename
-      Path.setup(filename).identifier_files
+      Path.setup(filename.dup).identifier_files
     else
       []
     end
@@ -442,16 +485,16 @@ end
   end
 
   def values_to_s(values)
-      case
-      when (values.nil? and fields.nil?)
-        "\n"
-      when (values.nil? and not fields.nil?)
-        "\t" << ([""] * fields.length) * "\t" << "\n"
-      when (not Array === values)
-        "\t" << values.to_s << "\n"
-      else
-        "\t" << values.collect{|v| Array === v ? v * "|" : v} * "\t" << "\n"
-      end
+    case
+    when (values.nil? and fields.nil?)
+      "\n"
+    when (values.nil? and not fields.nil?)
+      "\t" << ([""] * fields.length) * "\t" << "\n"
+    when (not Array === values)
+      "\t" << values.to_s << "\n"
+    else
+      "\t" << values.collect{|v| Array === v ? v * "|" : v} * "\t" << "\n"
+    end
   end
 
   def to_s(keys = nil, no_options = false)