rbbt-util 2.1.0 → 3.0.2

data/lib/rbbt/util/tsv/accessor.rb

@@ -1,3 +1,4 @@
+require 'rbbt/util/resource'
 require 'rbbt/util/misc'
 
 class TSV
@@ -68,12 +69,12 @@ class TSV
       when (TSV === identifiers.first or identifiers.empty?)
         identifiers
       when
-        identifiers.collect{|f| Path.path(f, datadir, namespace)}
+        identifiers.collect{|f| Resource::Path.path(f, nil, namespace)}
       end
     when (identifiers and not Array === identifiers)
-      [Path.path(identifiers, datadir)]
+      [Resource::Path.path(identifiers, nil, namespace)]
     when filename
-      Path.path(filename, datadir).identifier_files
+      Resource::Path.path(filename, nil, namespace).identifier_files
     else
       []
     end
@@ -176,7 +177,10 @@ class TSV
     if String === value && value =~ /__Ref:(.*)/
       return self[$1]
     else
-      value = NamedArray.name value, fields if Array === value and fields 
+
+      if Array === value and fields 
+        value = NamedArray.name value, fields 
+      end
       value
     end
   end
@@ -245,13 +249,18 @@ class TSV
   end
 
   def include?(key)
-    data.include? key
+    @data.include? key
   end
 
-  def to_s(keys = nil)
+  def to_s(keys = nil, no_options = false)
+    if FalseClass === keys or TrueClass === keys
+      no_options = keys
+      keys = nil
+    end
+
     str = ""
 
-    str << "#: " << Misc.hash2string(EXTRA_ACCESSORS.collect{|key| [key, self.send(key)]}) << "\n"
+    str << "#: " << Misc.hash2string(EXTRA_ACCESSORS.collect{|key| [key, self.send(key)]}) << "\n" unless no_options
     if fields
       str << "#" << key_field << "\t" << fields * "\t" << "\n"
     end

data/lib/rbbt/util/tsv/attach.rb

@@ -1,5 +1,182 @@
 class TSV
+  def self.paste_merge(file1, file2, output, sep = "\t")
+    case
+    when (String === file1 and not file1.index("\n") and file1.length < 250 and File.exists?(file1))
+      file1 = CMD.cmd("sort -k1,1 -t'#{sep}' #{ file1 } | grep -v '^#{sep}' ", :pipe => true)
+    when (String === file1 or StringIO === file1)
+      file1 = CMD.cmd("sort -k1,1 -t'#{sep}' | grep -v '^#{sep}'", :in => file1, :pipe => true)
+    end
+
+    case
+    when (String === file2 and not file2.index("\n") and file2.length < 250 and File.exists?(file2))
+      file2 = CMD.cmd("sort -k1,1 -t'#{sep}' #{ file2 } | grep -v '^#{sep}' ", :pipe => true)
+    when (String === file2 or StringIO === file2)
+      file2 = CMD.cmd("sort -k1,1 -t'#{sep}' | grep -v '^#{sep}'", :in => file2, :pipe => true)
+    end
+
+    output = File.open(output, 'w') if String === output
+
+    cols1 = nil
+    cols2 = nil
+
+    done1 = false
+    done2 = false
+
+    key1 = key2 = nil
+    while key1.nil?
+      while (line1 = file1.gets) =~ /#/; end
+      key1, *parts1 = line1.sub("\n",'').split(sep, -1)
+      cols1 = parts1.length
+    end
+
+    while key2.nil?
+      while (line2 = file2.gets) =~ /#/; end
+      key2, *parts2 = line2.sub("\n",'').split(sep, -1)
+      cols2 = parts2.length
+    end
+
+    key = key1 < key2 ? key1 : key2
+    parts = [""] * (cols1 + cols2)
+    while not (done1 and done2)
+      while (not done1 and key1 == key)
+        parts1.each_with_index do |part, i|
+          parts[i] = (parts[i].nil? or parts[i].empty?) ? part : parts[i] << "|" << part
+        end
+        key1 = nil
+        while key1.nil? and not done1
+          if file1.eof?; done1 = true; else key1, *parts1 = file1.gets.sub("\n",'').split(sep, -1) end
+        end
+      end
+      while (not done2 and key2 == key)
+        parts2.each_with_index do |part, i|
+          i += cols1
+          parts[i] = (parts[i].nil? or parts[i].empty?) ? part : parts[i] << "|" << part
+        end
+        key2 = nil
+        while key2.nil? and not done2
+          if file2.eof?; done2 = true; else key2, *parts2 = file2.gets.sub("\n",'').split(sep, -1) end
+        end
+      end
+
+      output.puts [key, parts].flatten * sep
+      parts = [""] * (cols1 + cols2)
+
+      case
+      when done1
+        key = key2
+      when done2
+        key = key1
+      else
+        key = key1 < key2 ? key1 : key2
+      end
+    end
+
+    output.close
+  end
+
+  def self.paste(file1, file2, output, sep = "\t")
+    case
+    when (String === file1 and not file1.index("\n") and file1.length < 250 and File.exists?(file1))
+      file1 = CMD.cmd("sort -k1,1 -t'#{sep}' #{ file1 } ", :pipe => true)
+    when String === file1
+      file1 = CMD.cmd("sort -k1,1 -t'#{sep}'", :in => file1, :pipe => true)
+    end
+
+    case
+    when (String === file2 and not file2.index("\n") and file2.length < 250 and File.exists?(file2))
+      file2 = CMD.cmd("sort -k1,1 -t'#{sep}' #{ file2 } ", :pipe => true)
+    when String === file2
+      file2 = CMD.cmd("sort -k1,1 -t'#{sep}'", :in => file2, :pipe => true)
+    end
+
+    output = File.open(output, 'w') if String === output
+
+    cols1 = nil
+    cols2 = nil
+
+    done1 = false
+    done2 = false
+
+    while (line1 = file1.gets) =~ /#/; end
+    line1.strip!
+    parts1 = line1.split(sep)
+    key1 = parts1.shift
+    cols1 = parts1.length
 
+    while (line2 = file2.gets) =~ /#/; end
+    line2.strip!
+    parts2 = line2.split(sep)
+    key2 = parts2.shift
+    cols2 = parts2.length
+    while not (done1 or done2)
+      case
+      when key1 < key2
+        output.puts [key1, parts1,  [""] * cols2] * sep
+        if file1.eof?
+          done1 = true
+        else
+          line1 = file1.gets
+          line1.strip!
+          parts1 = line1.split(sep)
+          key1 = parts1.shift
+        end
+      when key2 < key1
+        output.puts [key2,  [""] * cols1, parts2] * sep
+        if file2.eof?
+          done2 = true
+        else
+          line2 = file2.gets
+          line2.strip!
+          parts2 = line2.split(sep)
+          key2 = parts2.shift
+        end
+      when key1 == key2
+        output.puts [key1, parts1,  parts2] * sep
+        if file1.eof?
+          done1 = true
+        else
+          line1 = file1.gets
+          line1.strip!
+          parts1 = line1.split(sep)
+          key1 = parts1.shift
+        end
+        if file2.eof?
+          done2 = true
+        else
+          line2 = file2.gets
+          line2.strip!
+          parts2 = line2.split(sep)
+          key2 = parts2.shift
+        end
+      end
+    end
+
+    while not done1
+      output.puts [key1, parts1,  [""] * cols2] * sep
+      if file1.eof?
+        done1 = true
+      else
+        line1 = file1.gets
+        line1.strip!
+        parts1 = line1.split(sep)
+        key1 = parts1.shift
+      end
+    end
+
+    while not done2
+      output.puts [key2,  [""] * cols1, parts2] * sep
+      if file2.eof?
+        done2 = true
+      else
+        line2 = file2.gets
+        line2.strip!
+        parts2 = line2.split(sep)
+        key2 = parts2.shift
+      end
+    end
+
+    output.close
+  end
   #{{{ Attach Methods
   
   def attach_same_key(other, fields = nil)
@@ -26,7 +203,7 @@ class TSV
   def attach_source_key(other, source, fields = nil)
     fields = other.fields - [key_field].concat(self.fields) if fields.nil?
 
-    other = other.tsv unless TSV === other
+    other = other.tsv(:persistence => :no_create) unless TSV === other
     field_positions = fields.collect{|field| other.identify_field field}
     field_names     = field_positions.collect{|pos| pos == :key ? other.key_field : other.fields[pos] }
 
@@ -87,7 +264,11 @@ class TSV
           next unless other.include? source_key
           new_values = field_positions.collect do |pos|
             if pos == :key
-              source_key
+              if other.type == :double
+                [source_key]
+              else
+                source_key
+              end
             else
               other[source_key][pos]
             end
@@ -119,7 +300,10 @@ class TSV
   #{{{ Attach Helper
 
   # May make an extra index!
-  def self.find_path(files, in_namespace = false)
+  def self.find_path(files, options = {})
+    options      = Misc.add_defaults options, :in_namespace => false
+    in_namespace = options[:in_namespace]
+
     if in_namespace
       ids = [files.first.all_namespace_fields(in_namespace)]
       ids += files[1..-1].collect{|f| f.all_fields}
@@ -128,9 +312,6 @@ class TSV
     end
     id_list = []
 
-    ids.flatten.each do |field|
-    end
-
     ids.each_with_index do |list, i|
       break if i == ids.length - 1
       match = list.select{|field| 
@@ -148,8 +329,12 @@ class TSV
     end
   end
 
-  def self.build_traverse_index(files, in_namespace = false)
-    path = find_path(files, in_namespace)
+  def self.build_traverse_index(files, options = {})
+    options      = Misc.add_defaults options, :in_namespace => false, :persist_input => false
+    in_namespace = options[:in_namespace]
+    persist_input = options[:persist_input]
+
+    path = find_path(files, options)
 
     return nil if path.nil?
     
@@ -157,11 +342,10 @@ class TSV
     
     Log.medium "Found Traversal: #{traversal_ids * " => "}"
 
-    current_key = files.first.all_fields.first
-    target = files.last.all_fields.first
-    target = nil
     current_id, current_file = path.shift
-    index   = current_file.index :target => current_id, :fields =>  current_key, :persistence => false
+    current_key = current_file.all_fields.first
+
+    index   = current_file.index :target => current_id, :fields =>  current_key, :persistence => persist_input
 
     while not path.empty?
       current_id, current_file = path.shift
@@ -175,7 +359,10 @@ class TSV
     index
   end
 
-  def self.find_traversal(tsv1, tsv2, in_namespace = false)
+  def self.find_traversal(tsv1, tsv2, options = {})
+    options      = Misc.add_defaults options, :in_namespace => false
+    in_namespace = options[:in_namespace]
+
     identifiers1 = tsv1.identifier_files || []
     identifiers2 = tsv2.identifier_files || []
 
@@ -188,7 +375,7 @@ class TSV
       files1.push identifiers1.shift
       identifiers2.each_with_index do |e,i|
         files2 = identifiers2[(0..i)]
-        index  = build_traverse_index(files1 + files2.reverse, in_namespace)
+        index  = build_traverse_index(files1 + files2.reverse, options)
         return index if not index.nil?
       end
     end
@@ -197,8 +384,8 @@ class TSV
   end
 
   def attach(other, fields = nil, options = {})
-    options = Misc.add_defaults options, :in_namespace => true
-    in_namespace = Misc.process_options options, :in_namespace
+    options      = Misc.add_defaults options, :in_namespace => false
+    in_namespace = options[:in_namespace]
 
     fields = other.fields - [key_field].concat(self.fields) if fields == :all
     fields = other.fields_in_namespace - [key_field].concat(self.fields) if fields.nil?
@@ -211,7 +398,7 @@ class TSV
     when (in_namespace and self.fields_in_namespace.include?(other.key_field))
       attach_source_key other, other.key_field, fields
     else
-      index = TSV.find_traversal(self, other, in_namespace)
+      index = TSV.find_traversal(self, other, options)
       raise "Cannot traverse identifiers" if index.nil?
       attach_index other, index, fields
     end
@@ -225,4 +412,20 @@ class TSV
     reorder :key, detached_fields
   end
 
+  def paste(other, options = {})
+    tmpfile = TmpFile.tmp_file
+    TSV.paste(self.to_s, other.to_s, tmpfile)
+
+    new = TSV.new(tmpfile, options)
+
+    new.key_field = self.key_field unless self.key_field.nil?
+    if self.fields and other.fields
+      new.fields = self.fields + other.fields
+    end
+    
+    FileUtils.rm tmpfile if File.exists? tmpfile
+
+    new
+  end
+
 end

data/lib/rbbt/util/tsv/index.rb

@@ -232,6 +232,7 @@ class TSV
   end
 
   def self.field_matches(tsv, values)
+    values = [values] if not Array === values
     if values.flatten.sort[0..9].compact.collect{|n| n.to_i} == (1..10).to_a
       return {}
     end
@@ -247,7 +248,7 @@ class TSV
     if tsv.type == :double
       tsv.through do |key,entry_values|
         fields.zip(entry_values).each do |field,entry_field_values|
-          field_values[field].concat entry_field_values
+          field_values[field].concat entry_field_values unless entry_field_values.nil?
         end
       end
     else
@@ -271,6 +272,10 @@ class TSV
     TSV.field_matches(self, values)
   end
 
+  def guess_field(values)
+    field_matches(values).sort_by{|field, matches| matches.uniq.length}.last
+  end
+
   def sorted_index(pos_start = nil, pos_end = nil)
     raise "Please specify indexing fields" if (pos_start.nil? and fields.length > 2)
 

data/lib/rbbt/util/tsv/manipulate.rb

@@ -1,4 +1,3 @@
-
 class TSV
 
   def through(new_key_field = :key, new_fields = nil, &block)
@@ -177,7 +176,7 @@ class TSV
     new.filename  = filename
     new.case_insensitive  = case_insensitive
     
-    case
+   case
     when (method.nil? and block_given?)
       through do |key, values|
         new[key] = values if yield key, values
@@ -199,7 +198,7 @@ class TSV
       method = method.values.first
       case
       when (Array === method and (key == :key or key_field == key))
-        method.each{|item| if values = self[item]; then  new[item] = values; end}
+        method.each{|item| new[item] = self[item] if self.include? item}
       when Array === method
         through :key, key do |key, values|
           new[key] = self[key] if (values.flatten & method).any?
@@ -250,9 +249,9 @@ class TSV
   def add_field(name = nil)
     each do |key, values|
       new_values = yield(key, values)
-      new_values = [new_values] if type == :double and not Array == new_values
+      new_values = [new_values] if type == :double and not Array === new_values
 
-      self[key] = values + [yield(key, values)]
+      self[key] = values + [new_values]
     end
 
     self.fields = self.fields + [name] if fields != nil and name != nil

data/lib/rbbt/util/tsv/parse.rb

@@ -7,6 +7,7 @@ class TSV
     ## split with delimiter, do not remove empty
     fields = io.split(delimiter, -1)
 
+
     fields
   end
 
@@ -116,6 +117,8 @@ class TSV
     fix, exclude, select, grep = 
       Misc.process_options options, :fix, :exclude, :select, :grep 
 
+    exclude ||= Misc.process_options options, :reject if options.include? :reject
+
     #{{{ Process rest
     data = {}
     single = type.to_sym != :double
@@ -152,9 +155,10 @@ class TSV
       if single
         ids = parse_fields(parts[key_pos], sep2)
         ids.collect!{|id| id.downcase} if case_insensitive
+        ids = ids.reject{|_id| _id.empty?}.uniq
         
         id = ids.shift
-        ids.each do |id2| data[id2] = "__Ref:#{id}"  end
+        ids.each do |id2| data[id2] = "__Ref:#{id}" unless data.include? id2 end
 
         next if data.include?(id) and type != :flat
 
@@ -171,8 +175,8 @@ class TSV
 
         extra.collect! do |elem| 
           case
-            when String === cast
-              elem.send(cast)
+            when (String === cast or Symbol === cast)
+              elem.send(cast.to_s)
             when Proc === cast
               cast.call elem
             end
@@ -195,9 +199,26 @@ class TSV
       else
         ids = parse_fields(parts[key_pos], sep2)
         ids.collect!{|id| id.downcase} if case_insensitive
+        ids = ids.reject{|_id| _id.empty?}.uniq
+
+        next if ids.empty?
 
         id = ids.shift
-        ids.each do |id2| data[id2] = "__Ref:#{id}"  end
+        while data.include? id and data[id] =~ /__Ref:(.*)/
+          data[id] = data[$1].collect{|e| e.dup}
+        end
+
+        all_ids = [id]
+        ids.each do |id2| 
+          if data.include? id2
+            while data[id2] =~ /__Ref:(.*)/ 
+              data[id2] = data[$1].collect{|e| e.dup}
+            end
+            all_ids << id2
+          else
+            data[id2] = "__Ref:#{id}" 
+          end
+        end
 
         if other_pos.nil? or (fields == nil and type == :flat)
           other_pos    = (0..(parts.length - 1)).to_a
@@ -207,31 +228,34 @@ class TSV
         extra = parts.values_at(*other_pos).collect{|f| parse_fields(f, sep2)}
         extra.collect! do |list| 
           case
-          when String === cast
-            list.collect{|elem| elem.send(cast)}
+          when (String === cast or Symbol === cast)
+            list.collect{|elem| elem.send(cast.to_s)}
           when Proc === cast
             list.collect{|elem| cast.call elem}
           end
         end if cast
 
         max_cols = extra.size if extra.size > (max_cols || 0)
-        if not merge
-          data[id] = extra unless data.include? id
-        else
-          if not data.include? id
-            data[id] = extra
+
+        all_ids.each do |id|
+          if not merge
+            data[id] = extra unless data.include? id
           else
-            entry = data[id]
-            while entry =~ /__Ref:(.*)/ do entry = data[$1] end
-            extra.each_with_index do |f, i|
-              if f.empty?
-                next unless keep_empty
-                f= [""]
+            if not data.include? id
+              data[id] = extra
+            else
+              entry = data[id]
+              while entry =~ /__Ref:(.*)/ do entry = data[$1] end
+              extra.each_with_index do |f, i|
+                if f.empty?
+                  next unless keep_empty
+                  f= [""]
+                end
+                entry[i] ||= []
+                entry[i] = entry[i].concat f
               end
-              entry[i] ||= []
-              entry[i] = entry[i].concat f
+              data[id] = entry
             end
-            data[id] = entry
           end
         end
       end
@@ -254,7 +278,7 @@ class TSV
 
     fields = nil if Fixnum === fields or (Array === fields and fields.select{|f| Fixnum === f}.any?)
     fields ||= other_fields
-    [data, {:key_field => key_field, :fields => fields, :type => type, :case_insensitive => case_insensitive, :namespace => namespace, :datadir => options[:datadir], :identifiers => options[:identifiers], :cast => !!cast}]
+    [data, {:key_field => key_field, :fields => fields, :type => type, :case_insensitive => case_insensitive, :namespace => namespace, :datadir => options[:datadir], :identifiers => options[:identifiers], :cast => (cast.nil? ? false : cast)}]
   end
 
 end