rbbt-util 1.2.1 → 2.0.1

data/lib/rbbt/util/tsv/parse.rb

@@ -0,0 +1,258 @@
+require 'rbbt/util/misc'
+class TSV
+ 
+  def self.parse_fields(io, delimiter = "\t")
+    return [] if io.nil?
+
+    ## split with delimiter, do not remove empty
+    fields = io.split(delimiter, -1)
+
+    fields
+  end
+
+  def self.parse_header(stream, sep = nil, header_hash = nil)
+    sep = /\t/ if sep.nil?
+    header_hash = "#" if header_hash.nil?
+
+    fields, key_field = nil
+    options = {}
+
+    # Get line
+
+    line = stream.gets
+    raise "Empty content" if line.nil?
+
+    # Process options line
+    
+    if line and line =~ /^#{header_hash}: (.*)/
+      options = Misc.string2hash $1
+      line = stream.gets
+    end
+
+    # Determine separator
+    
+    sep = options[:sep] if options[:sep]
+
+    # Process fields line
+
+    if line and line =~ /^#{header_hash}/
+      line.chomp!
+      fields = parse_fields(line, sep)
+      key_field = fields.shift
+      key_field = key_field[(0 + header_hash.length)..-1] # Remove initial hash character
+      line = stream.gets
+    end
+
+    # Return fields, options and first line
+
+    return key_field, fields, options, line
+  end
+
+  def self.parse(stream, options = {})
+
+    # Prepare options
+    
+    key_field, other_fields, more_options, line = TSV.parse_header(stream, options[:sep], options[:header_hash])
+
+    options = Misc.add_defaults options, more_options
+
+    options = Misc.add_defaults options, 
+      :case_insensitive => false,
+      :type             => :double,
+      :namespace        => nil,
+      :identifiers      => nil,
+
+      :merge            => false,
+      :keep_empty       => (options[:type] != :flat and options[:type] != :single),
+      :cast             => nil,
+
+      :header_hash      => '#',
+      :sep              => "\t",
+      :sep2             => "|",
+
+      :key              => 0,
+      :fields           => nil,
+
+      :fix              => nil,
+      :exclude          => nil,
+      :select           => nil,
+      :grep             => nil
+    
+    header_hash, sep, sep2 =
+      Misc.process_options options, :header_hash, :sep, :sep2
+
+    key, fields =
+      Misc.process_options options, :key, :fields
+
+    if key_field.nil?
+      key_pos      = key
+      key_field, fields = nil
+    else
+      all_fields = [key_field].concat other_fields
+
+      key_pos   = Misc.field_position(all_fields, key)
+
+      if String === fields or Symbol === fields
+        fields = [fields]
+      end
+
+      if fields.nil?
+        other_pos    = (0..(all_fields.length - 1)).to_a
+        other_pos.delete key_pos
+      else
+        if Array === fields
+          other_pos = fields.collect{|field| Misc.field_position(all_fields, field)}
+        else
+          other_pos = Misc.field_position(all_fields, fields)
+        end
+      end
+
+      key_field = all_fields[key_pos]
+      fields    = all_fields.values_at *other_pos
+    end
+
+    case_insensitive, type, namespace, merge, keep_empty, cast = 
+      Misc.process_options options, :case_insensitive, :type, :namespace, :merge, :keep_empty, :cast
+    fix, exclude, select, grep = 
+      Misc.process_options options, :fix, :exclude, :select, :grep 
+
+    #{{{ Process rest
+    data = {}
+    single = type.to_sym != :double
+    max_cols = 0
+    while line do
+      line.chomp!
+
+      if line.empty?                           or
+         (exclude and     exclude.call(line))  or
+         (select  and not select.call(line))
+
+         line = stream.gets
+         next
+      end
+
+      line = fix.call line if fix
+      break if not line
+
+
+      if header_hash and not header_hash.empty? and line =~ /^#{header_hash}/
+        line = stream.gets
+        next
+      end
+
+      # Chunk fields
+      parts = parse_fields(line, sep)
+
+      # Get next line
+      line = stream.gets
+
+      # Get id field
+      next if parts[key_pos].nil? || parts[key_pos].empty?
+     
+      if single
+        ids = parse_fields(parts[key_pos], sep2)
+        ids.collect!{|id| id.downcase} if case_insensitive
+        
+        id = ids.shift
+        ids.each do |id2| data[id2] = "__Ref:#{id}"  end
+
+        next if data.include?(id) and type != :flat
+
+        if key_field.nil?
+          other_pos    = (0..(parts.length - 1)).to_a
+          other_pos.delete key_pos
+        end
+
+        if type == :flat 
+          extra = parts.values_at(*other_pos).collect{|f| parse_fields(f, sep2)}.flatten
+        else
+          extra = parts.values_at(*other_pos).collect{|f| parse_fields(f, sep2).first}
+        end
+
+        extra.collect! do |elem| 
+          case
+            when String === cast
+              elem.send(cast)
+            when Proc === cast
+              cast.call elem
+            end
+        end if cast
+
+        case
+        when type == :single
+            data[id] = extra.first
+        when type == :flat
+          if data.include? id
+            data[id].concat extra
+          else
+            data[id] = extra 
+          end
+        else
+          data[id] = extra 
+        end
+
+        max_cols = extra.size if extra.size > (max_cols || 0) unless type == :flat
+      else
+        ids = parse_fields(parts[key_pos], sep2)
+        ids.collect!{|id| id.downcase} if case_insensitive
+
+        id = ids.shift
+        ids.each do |id2| data[id2] = "__Ref:#{id}"  end
+
+        if key_field.nil?
+          other_pos    = (0..(parts.length - 1)).to_a
+          other_pos.delete key_pos
+        end
+
+        extra = parts.values_at(*other_pos).collect{|f| parse_fields(f, sep2)}
+        extra.collect! do |list| 
+          case
+          when String === cast
+            list.collect{|elem| elem.send(cast)}
+          when Proc === cast
+            list.collect{|elem| cast.call elem}
+          end
+        end if cast
+
+        max_cols = extra.size if extra.size > (max_cols || 0)
+        if not merge
+          data[id] = extra unless data.include? id
+        else
+          if not data.include? id
+            data[id] = extra
+          else
+            entry = data[id]
+            while entry =~ /__Ref:(.*)/ do entry = data[$1] end
+            extra.each_with_index do |f, i|
+              if f.empty?
+                next unless keep_empty
+                f= [""]
+              end
+              entry[i] ||= []
+              entry[i] = entry[i].concat f
+            end
+            data[id] = entry
+          end
+        end
+      end
+    end
+
+    if keep_empty and max_cols > 0
+      data.each do |key, values| 
+        next if values =~ /__Ref:/
+        new_values = values
+        max_cols.times do |i|
+          if type == :double
+            new_values[i] = [""] if new_values[i].nil? or new_values[i].empty?
+          else
+            new_values[i] = "" if new_values[i].nil?
+          end
+        end
+        data[key] = new_values
+      end
+    end
+
+    [data, {:key_field => key_field, :fields => fields, :type => type, :case_insensitive => case_insensitive, :namespace => namespace, :datadir => options[:datadir], :identifiers => options[:identifiers], :cast => !!cast}]
+  end
+
+end

data/share/lib/R/util.R

@@ -18,12 +18,15 @@ rbbt.tsv <- function(filename, sep = "\t", comment.char ="#",  ...){
   data=read.table(file=filename, sep=sep, fill=TRUE,  as.is=TRUE, row.names=1, comment.char = comment.char, ...);
   f = file(filename, 'r');
   headers = readLines(f, 1);
-  close(f);
-  if (grep(paste("^", comment.char, sep=""), headers)){
+  if (length(grep("^#: ", headers)) > 0){ 
+      headers = readLines(f, 1); 
+  } 
+  if (length(grep("^#", headers)) > 0){
       fields = strsplit(headers, sep)[[1]];
       fields = fields[2:length(fields)];
       names(data) <- fields;
   }
+  close(f);
   return(data);
 }
 
@@ -36,7 +39,6 @@ rbbt.tsv.write <- function(filename, data, key.field = NULL){
   for (name in colnames(data)){ header = paste(header, name, sep="\t");}
   header = paste(header, "\n", sep="");
   cat(header, file=f);
-  cat(header, file=stderr());
   
   close(f);
 

data/test/rbbt/util/test_R.rb

@@ -3,7 +3,15 @@ require 'rbbt/util/R'
 
 class TestR < Test::Unit::TestCase
   def test_sum
-    assert_equal "6", R.run('cat(3+3)').read
+    assert_equal "6", R.run('cat(3+3)').read.split(/\n/).last
+  end
+
+  def test_tsv_R
+    tsv = TSV.new({:a => 1, :b => 2})
+    tsv2 = tsv.R <<-EOF
+data = data + 1
+    EOF
+    puts tsv2.to_s
   end
 end
 

data/test/rbbt/util/test_data_module.rb

@@ -8,6 +8,7 @@ SHAREDIR = File.join(PKGData.sharedir_for_file(__FILE__), 'install/DataTest')
 FileUtils.mkdir_p SHAREDIR
 File.open(File.join(SHAREDIR, 'Rakefile'), 'w') do |f|
   f.puts "file :file1 do |t| File.open(t.name, 'w') do |f| f.write 'File 1' end end"
+  f.puts "file :tsv_file do |t| File.open(t.name, 'w') do |f| f.write 'a\t1\nb\t2\n' end end"
 end
 
 module DataTest  
@@ -27,14 +28,18 @@ class TestDataModule < Test::Unit::TestCase
     FileUtils.mkdir_p SHAREDIR
     File.open(File.join(SHAREDIR, 'Rakefile'), 'w') do |f|
       f.puts "file :file1 do |t| File.open(t.name, 'w') do |f| f.write 'File 1' end end"
+      f.puts "file :tsv_file do |t| File.open(t.name, 'w') do |f| f.write 'a\t1\nb\t2\n' end end"
     end
   end
 
   def test_rakefile
+    assert_equal Rbbt.files.DataTest, DataTest.datadir
     assert_equal "File 1", Rbbt.files.DataTest.file1.read
     assert_equal "Hello world", DataTest.salute("world")
     assert_equal "Hello world", DataTest::with_key("world").salute
     assert_equal "Hello world", DataTest::World.salute
+    assert_equal "DataTest", Rbbt.files.DataTest.tsv_file.namespace
+    assert_equal "DataTest", Rbbt.files.DataTest.tsv_file.tsv.namespace
     FileUtils.rm_rf File.join(Rbbt.datadir, 'DataTest')
   end
 

data/test/rbbt/util/test_fix_width_table.rb

@@ -0,0 +1,107 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/util/fix_width_table'
+require 'rbbt/util/tsv'
+
+class TestFixWidthTable < Test::Unit::TestCase
+  def load_data(data)
+    Log.debug("Data:\n#{Open.read(data)}")
+    tsv = TSV.new(data, :list, :sep=>":", :cast => proc{|e| e =~ /(\s*)(_*)/; ($1.length..($1.length + $2.length - 1))})
+    tsv.add_field "Start" do |key, values|
+      values["Range"].first
+    end
+    tsv.add_field "End" do |key, values|
+      values["Range"].last
+    end
+
+    tsv = tsv.slice ["Start", "End"]
+ 
+    tsv
+  end
+
+  def test_options
+    TmpFile.with_file do |filename|
+      f = FixWidthTable.new filename, 100, true
+      f.close
+
+      f1 = FixWidthTable.new filename, 100, false
+
+      assert_equal true, f1.range
+    end
+  end
+
+  def test_add
+    TmpFile.with_file do |filename|
+      f = FixWidthTable.new filename, 100, true
+      f.add [1,2,0], "test1"
+      f.add [3,4,0], "test2"
+      f.read
+
+      assert_equal 1, f.pos(0)
+      assert_equal 3, f.pos(1)
+      assert_equal 2, f.pos_end(0)
+      assert_equal 4, f.pos_end(1)
+      assert_equal 0, f.overlap(0)
+      assert_equal 0, f.overlap(1)
+      assert_equal "test1", f.value(0)
+      assert_equal "test2", f.value(1)
+    end
+ 
+  end
+
+  def test_point
+    data =<<-EOF
+#: :sep=/\\s+/#:type=:single#:cast=to_i
+#ID Pos
+a 1
+b 10
+c 20
+d 12
+e 26
+f 11
+g 25
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      tsv = TSV.new datafile
+      ddd tsv
+      TmpFile.with_file do |filename|
+        f = FixWidthTable.new filename, 100, false
+        f.add_point tsv
+        f.read
+
+        assert_equal %w(), f[0].sort
+        assert_equal %w(b), f[10].sort
+        assert_equal %w(a b c d f), f[(0..20)].sort
+      end
+    end
+  end
+
+  def test_range
+    data =<<-EOF
+#ID:Range
+#:012345678901234567890
+a:   ______
+b: ______
+c:    _______
+d:  ____
+e:    ______
+f:             ___
+g:         ____
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      tsv = TSV.new load_data(datafile)
+      TmpFile.with_file do |filename|
+        f = FixWidthTable.new filename, 100, true
+        f.add_range tsv
+        f.read
+
+        assert_equal %w(), f[0].sort
+        assert_equal %w(b), f[1].sort
+        assert_equal %w(), f[20].sort
+        assert_equal %w(), f[(20..100)].sort
+        assert_equal %w(a b d), f[3].sort
+        assert_equal %w(a b c d e), f[(3..4)].sort
+      end
+    end
+  end
+end
+

data/test/rbbt/util/test_misc.rb

@@ -32,4 +32,47 @@ class TestMisc < Test::Unit::TestCase
     assert_equal(1, a['a'])
   end
 
+  def test_path_relative_to
+    assert_equal "test/foo", Misc.path_relative_to('test/test/foo', 'test')
+  end
+
+  def test_chunk
+    test =<<-EOF
+This is an example file. Entries are separated by Entry
+-- Entry
+1
+2
+3
+-- Entry
+4
+5
+6
+    EOF
+
+    assert_equal "1\n2\n3", Misc.chunk(test, /^-- Entry/).first.strip
+  end
+
+  def test_hash2string
+    hash = {}
+    assert_equal hash, Misc.string2hash(Misc.hash2string(hash))
+
+    hash = {:a => 1}
+    assert_equal hash, Misc.string2hash(Misc.hash2string(hash))
+ 
+    hash = {:a => true}
+    assert_equal hash, Misc.string2hash(Misc.hash2string(hash))
+
+    hash = {:a => Misc}
+    assert_equal hash, Misc.string2hash(Misc.hash2string(hash))
+ 
+    hash = {:a => :b}
+    assert_equal hash, Misc.string2hash(Misc.hash2string(hash))
+ 
+    hash = {:a => /test/}
+    assert_equal({}, Misc.string2hash(Misc.hash2string(hash)))
+ 
+
+
+ end
+
 end