bio-table 0.8.0 → 0.9.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 208de532730ef88e9a1ecc2961f08ceaa464d846
+  data.tar.gz: 4b88172ae40141a1db2dcb5c3a7b55daeea1beed
+SHA512:
+  metadata.gz: 5d0a5f044e44a89fd047db4443fca2d8729af55f0cc7106ab45e939e3e92c7356dffd7f9308a63cb4a0ff39b313c9321edf3cb637af587f8b7ff8796c1083415
+  data.tar.gz: cb9abfe846b2c1949d027e38415e486719cf59db5fb99d38ec495b1b828d8a838dca19ee63c302844f82fb23dc4ec2a61c1749cbfb286f77ac0eecffa14e470e

data/.travis.yml

@@ -1,12 +1,8 @@
 language: ruby
 rvm:
-  - 1.9.2
   - 1.9.3
-  - rbx-19mode
-#  - jruby-19mode # JRuby in 1.9 mode
-#  - 1.8.7
-#  - jruby-18mode # JRuby in 1.8 mode
-#  - rbx-18mode
-
+  - 2.1.0 
+  - ruby-head
+#  - jruby-head
 # uncomment this line if your project needs to run something other than `rake`:
 # script: bundle exec rspec spec

data/Gemfile

@@ -8,12 +8,11 @@ gem "bio-logger"
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do
-  gem "rspec", "~> 2.8.0"
-  gem "rdoc", "~> 3.12"
-  gem "cucumber", ">= 0"
-  gem "bundler", "> 1.0.0"
-  gem "jeweler", "~> 1.8.3"
+  gem "rspec"
+  gem "cucumber"
+  gem "bundler"
+  gem "jeweler", "~> 2.0.0"
   gem "bio", ">= 1.4.2"
-  gem "rdoc", "~> 3.12"
+  gem "rdoc"
   gem "regressiontest", ">= 0.0.2"
 end

data/README.md

@@ -46,6 +46,7 @@ Features:
 * Convert key-value (attributes) to RDF (nyi)
 * Convert table to JSON/YAML/XML (nyi)
 * Transpose matrix (nyi)
+* Convert a FASTA file to a table
 * etc. etc.
 
 and bio-table is pretty fast. To convert a 3Mb file of 18670 rows
@@ -86,7 +87,10 @@ When you have a special file format, it is also possible to use a string or rege
     bio-table --in-format regex --split-on '\s*,\s*' file
 ```
 
-To filter out rows that contain certain values
+To filter out rows that contain certain values, i.e., filter on the
+third column that have values less than 0.05 (this is actually the 5th
+column in a tabular file, where the fist column is the row name and
+the others count from zero).
 
 ```sh
     bio-table table1.csv --num-filter "values[3] <= 0.05" 
@@ -135,7 +139,7 @@ which takes the first 13 fields and compact removes the nil values.
 To filter out all rows with more than 3 NA values:
 
 ```sh
-  bio-table table.csv --num-filter 'values.to_a.size - values.compact.size > 3'
+  bio-table table.csv --num-filter 'values.size - values.compact.size > 3'
 ```
 
 Also string comparisons and regular expressions can be used. E.g.
@@ -201,6 +205,21 @@ again
 where we rewrite the rowname in capitals, and set the second field to
 empty if the third field is below 0.25. 
 
+Say we need a log transform, we can also transform and rewrite a full matrix with:
+
+```sh
+    bio-table table1.csv --rewrite 'fields = fields.map { |f| Math::log(f.to_f) }'
+```
+
+Note that 'fields' is an alias for 'field', but do not use them in the same expression. 
+Another option is to use (lazy) values:
+
+```sh
+    bio-table table1.csv --rewrite 'fields = values.map { |v| Math::log(v) }'
+```
+
+which saves the typing to to_f.
+
 ### Statistics
 
 bio-table can handle some column statistics using the Ruby statsample
@@ -210,7 +229,7 @@ gem
     gem install statsample
 ```
 
-(statsample is not loaded by default, as it has a host of
+(statsample is not loaded by default because it has a host of
 dependencies)
 
 Thereafter, to calculate the stats for columns 1 and 2 (rowname is column 0)
@@ -247,7 +266,14 @@ You can combine/concat two or more tables by passing in multiple file names
 ```
 
 this will append table2 to table1, assuming they have the same headers
-(you can use the --columns switch!)
+(you can use the --columns switch at the same time!). With --skip
+the header lines are skipped in every file. This can be a real asset
+when using the Unix split command on input files and combining output 
+files again. Something this might work:
+
+```sh
+    ls run/*.out -1|sort|xargs bio-table --skip 3
+```
 
 To combine tables side by side use the --merge switch:
 
@@ -261,6 +287,7 @@ with NA's, unless you add a filter, e.g.
 
 ```sh
     bio-table --merge table1.csv table2.csv --num-filter "values.compact.size == values.size"
+
 ```
 
 ### Splitting a table
@@ -310,7 +337,19 @@ finds the overlapping rows, based on the content of column 2.
 bio-table currently reads comma separated files and tab delimited
 files.
 
-(more soon)
+bio-table can also parse a FASTA file and turn it into a table using
+a flexible regular expression to fetch the IDs
+
+```sh
+    bio-table --fasta '^(\S+)' test/data/input/aa.fa
+```
+
+notice the parentheses - these capture the ID and create the first 
+column. If two captures are defined another column gets added. Try
+
+```sh
+    bio-table --fasta '^(\S+).*?(\d+) aa' test/data/input/aa.fa
+```
 
 ### Using STDIN
 

data/VERSION

@@ -1 +1 @@
-0.8.0
+0.9.0

data/bin/bio-table

@@ -38,7 +38,6 @@ options[:show_help] = true if ARGV.size == 0 and not INPUT_ON_STDIN
 opts = OptionParser.new do |o|
   o.banner = "Usage: #{File.basename($0)} [options] filename\n\n"
 
- 
   o.on('--num-filter expression', 'Numeric filtering function') do |par|
     options[:num_filter] = par
   end
@@ -130,6 +129,10 @@ opts = OptionParser.new do |o|
     options[:evaluate] = s
   end
 
+  o.on("--fasta regex",String,"Read FASTA format creating ID with regex") do | regex |
+    options[:fasta] = regex
+  end
+
   o.on('--blank-nodes','Output (RDF) blank nodes - allowing for duplicate row names') do 
     options[:blank_nodes] = true
   end
@@ -137,7 +140,7 @@ opts = OptionParser.new do |o|
   o.on('--statistics','Output column statistics') do 
     options[:statistics] = true
   end
-
+ 
   o.separator "\n\tVerbosity:\n\n"
   
   o.on("--logger filename",String,"Log to file (default stderr)") do | name |
@@ -183,7 +186,7 @@ end
 
 Bio::Log::CLI.configure('bio-table')
 logger = Bio::Log::LoggerPlus['bio-table']
-logger.info [options, ARGV]
+logger.info [options]
 
 include BioTable
 
@@ -207,6 +210,17 @@ if options[:overlap]
   exit
 end
 
+if options[:fasta]
+  logger.warn "Column settings are ignored for --fasta" if options[:columns]
+  ARGV.each do | fn |
+    print "id\tseq\n"
+    FastaReader.new(fn,options[:fasta]).each do | rec |
+      print rec.id,"\t",rec.seq,"\n"
+    end
+  end
+  exit
+end
+
 if options[:merge]
   ts = []
   ARGV.each do | fn |
@@ -233,10 +247,12 @@ writer =
 
 if INPUT_ON_STDIN
   opts = options.dup # so we can 'safely' modify options
+  has_input = false
   BioTable::TableLoader.emit(STDIN, opts).each do |row, type| 
     writer.write(TableRow.new(row[0],row[1..-1]),type)
+    has_input = true
   end
-  options[:write_header] = false  # don't write the header for chained files
+  options[:write_header] = false if has_input  # don't write the header for chained files
 end
 
 statistics = if options[:statistics]

data/lib/bio-table.rb

@@ -26,6 +26,7 @@ require 'bio-table/diff.rb'
 require 'bio-table/overlap.rb'
 require 'bio-table/merge.rb'
 require 'bio-table/rdf.rb'
+require 'bio-table/parsers/fastareader.rb'
 
 module BioTable
   autoload :Statistics,'bio-table/statistics'

data/lib/bio-table/filter.rb

@@ -1,6 +1,10 @@
 module BioTable
 
+  # LazyValues fetches values on demand from the @fields array. In the [] method
+  # a field is transformed into a float when it is called.
+
   class LazyValues
+
     include Enumerable
 
     def initialize fields
@@ -16,12 +20,16 @@ module BioTable
       @values[index]
     end
 
-    def each
-      @fields.each_with_index do | field, i |
-        yield self[i]
+    def each &block
+      @fields.each_with_index do |field,i|
+        if block_given?
+          block.call self[i]
+        else
+          yield self[i]
+        end
       end
     end
-
+  
     def compact
       a = []
       each do | e |
@@ -29,6 +37,10 @@ module BioTable
       end
       a
     end
+
+    def size
+      @fields.size
+    end
   end
 
   module Filter

data/lib/bio-table/parsers/fastareader.rb

@@ -0,0 +1,141 @@
+# FastaReader (originally from BigBio)
+#
+
+
+class FastaRecord
+  attr_accessor :id, :descr, :seq
+
+  def initialize id, descr, seq
+    @id = id
+    @descr = descr
+    @seq = seq
+  end
+end
+
+class FastaReader
+
+  # Initalize the reader of FASTA file _fn_. Options can be :regex and
+  # :index (true/false)
+  def initialize fn, regex = nil
+    @logger = Bio::Log::LoggerPlus['bio-table']
+    @f = File.open(fn)
+    @fread_once = false
+    @regex = regex
+    @regex = '^(\S+)' if @regex == nil
+    @regex = '('+regex+')' if regex !~ /\(/
+    @logger.info "Parsing FASTA with ID regex '"+@regex+"'"
+  end
+
+  # Parse the FASTA file and yield id, descr, sequence. When the indexer is on
+  # it will index the records the first time. Note that, with indexing, when
+  # you don't complete parsing there will be an error the second time. This is
+  # a  # trade-off, otherwise one would always have to index the file and read
+  # it twice.
+  def parse_each
+    @f.seek 0    # force file rewind
+    @rec_fpos = 0
+    @rec_line = @f.gets
+    fpos = 0
+    @count = 0
+    begin
+      # digest id from record description
+      id, descr = digest_tag(@rec_line)
+      id_fpos = @rec_fpos
+      # parse the sequence
+      seq = ""
+      begin
+        fpos = @f.tell
+        line = @f.gets
+        break if line =~ /^>/
+        seq += line.strip 
+      end while !@f.eof 
+      # new record
+      @count += 1
+      @rec_fpos = fpos
+      @rec_line = line
+      # p [@rec_line, id, id_fpos]
+      # indexer_set(id, id_fpos) if @indexer and not @fread_once
+      yield id, descr, seq
+    end while !@f.eof
+    @fread_once = true
+  end
+
+  # returns a FastaRecord for every item (invokes parse_each)
+  def each
+    parse_each { | id, descr, seq | yield FastaRecord.new(id, descr, seq) }
+  end
+
+  def first
+    parse_each { | id, descr, seq | 
+      return FastaRecord.new(id, descr, seq) 
+    }
+  end
+
+  # Return a record by its +id+, nil when not found
+  def get id
+    indexed?
+    if fpos = indexer_get(id)
+      get_rec(fpos)
+    else
+      nil
+    end
+  end
+
+  def get_rec fpos
+    @f.seek fpos
+    tag = @f.gets
+    seq = ""
+    begin
+      line = @f.gets
+      break if line =~ /^>/
+      seq += line.strip 
+    end while !@f.eof
+    id, descr = digest_tag(tag)
+    FastaRecord.new(id,descr,seq)
+  end
+
+  def get_by_index idx
+    indexed?
+    if fpos = indexer_get_by_index(idx)[1]
+      ret = get_rec(fpos)
+      return ret
+    end
+    nil
+  end
+
+  def digest_tag tag
+    if tag =~ /^>/
+      descr = $'.strip
+      matches = /#{@regex}/.match(descr).captures
+      if matches.size > 0
+        # p matches
+        return matches.join("\t"), descr
+      end
+      p descr  # do not remove these
+      p @regex
+    end
+    raise "Can not digest '#{tag}' using '"+@regex+"'"
+  end
+
+  # Returns the size of the dataset - as read. After the final
+  # record the size represents the number of items in the FASTA file
+  def size
+    @count
+  end
+
+  def close
+    @f.close
+  end
+
+  private
+
+  def indexed?
+    if @indexer and not @fread_once
+      # force indexer
+      # $stderr.print "Force indexer"
+      parse_each { | x, y, z | nil }
+    end
+    true
+  end
+
+end

data/lib/bio-table/rewrite.rb

@@ -2,7 +2,11 @@ module BioTable
 
   module Rewrite
 
+    # Rewrite fields. Both field and fields can be used, but not at the same time.
     def Rewrite::rewrite code, rowname, field
+      fields = field
+      original = field
+      values = LazyValues.new(field)
       return rowname,field if not code or code==""
       begin
         eval(code)
@@ -10,6 +14,7 @@ module BioTable
         $stderr.print "Failed to evaluate ",rowname," ",field," with ",code,"\n"
         raise 
       end
+      field = fields if fields != original
       return rowname,field
     end
   end

data/lib/bio-table/statistics.rb

@@ -3,7 +3,12 @@ module BioTable
 
   module Statistics
 
-require 'statsample'
+begin
+  require 'statsample'
+rescue LoadError
+  $stderr.print "Error: Missing statsample. Install with command 'gem install statsample'\n"
+  exit 1
+end
 
     attr_reader :columns
 

data/lib/bio-table/table_apply.rb

@@ -29,6 +29,7 @@ module BioTable
       @include_rownames = options[:with_rownames]
       @logger.debug "Include row names" if @include_rownames
       @first_column = (@include_rownames ? 0 : 1)
+      @write_header = options[:write_header]
     end
 
     def parse_header(line, options)  
@@ -39,7 +40,7 @@ module BioTable
       if options[:unshift_headers]
         header.unshift("ID")
       end
-      @logger.info(header) if @logger
+      @logger.info(header) if @logger and @write_header 
       header
     end
 

data/test/data/input/aa.fa

@@ -0,0 +1,7 @@
+>PITG_20587T0 | PITG_20587 | Phytophthora infestans cysteine protease family C48, putative (translation) (349 aa) | paired
+MQLRALLRDNDVCDVVSTLWMIMPGVREVWSFLTTFPVNKNGEGRSIQWRVDGDYVPDRVRFRLVESLVDDASDKLRGGLALDEEIELDSDGERMSSIESYVVSIEKVGQFTREQLEAMKSLWGLQDTCRNAVLCCTWLNSTVKPAVSDPASAGIIMGKILECWPYTSLVGFGFDLTYNNLFCFRDSAWLNDNAMRAFAVSKDAKNGTQPKATKSRISTSTLDKVGESVASHQFVLLPINFGGTHWGCLVVDRDTKVIKMYDSMGGKRNKKRLQKMAEEIRTGPLRDDSYEALEVTEPVQTNSDSCGVFVCRFFWTCVSSESPSDVSPAGITKLRWEMLHAVTKLRPR*
+>PITG_04498T0 | PITG_04498 | Phytophthora infestans cysteine protease family C48, putative (translation) (337 aa) | paired
+MKLRALLRDNDVCDVVSTLWMIMPGVREVGSFLTTFPLNKNGEGRSIQWRVDGDYVPDRVRFRLVESLVDDALDKLRGGLALDEEIELDRDGERMGSIESYVVSIEKVGQFTREQLEAMKSLWGLQDTCRNAVLCCTWLNSTVKPACWPYTPLVGFGFDLTYNNLFCFRDSAWLNDNAMRAFAVCLARYKNNCTVVIPPPKKAKDAKNGTQPKATKSRISTSTLDKVGESVASHQFVLLPINFGGTHWGCLVVKRDTKVIKMYDSMGGKRNKKRLQKMAEEIRTGPLRDDSYEALEVTEPVQTDSDSCGVFVDVSPAGITKLRWEMLHAVMKLRPR*
+>PITG_10111T0 | PITG_10111 | Phytophthora infestans cysteine protease family C48, putative (translation) (332 aa) | paired
+MKLRALLRDNDVCDVVSTLWMIMPGVREVGSFLTTFPVNKNGEGRSIQWRVDGDYVPDRVRFRLVESLVDDASDKLRGGLALDEEIELDTGQLTREQLEAMKSLWGLQDTCRNAVLCCTWLNSTILECWPYTPLVGFGFDLTYTNLFCFRDSAWLNDNAMRAFAVCLARYKNNCTVVIPPPQKAKDAKNGTQPKATKSRISTSTLDKVGESVASHQFVLLPINFGGTHWGCLVVDRDTKVVKMYDSMGGKRNKKRLEKMAEEIRTGPLRDDSYKALEVTEPVQTDSDSCGVFVCRFFWTCVSSESPSDVSPAGITKLRWEMLHAVMKLRPR*
+

metadata

@@ -1,115 +1,127 @@
 --- !ruby/object:Gem::Specification
 name: bio-table
 version: !ruby/object:Gem::Version
-  version: 0.8.0
-  prerelease: 
+  version: 0.9.0
 platform: ruby
 authors:
 - Pjotr Prins
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-01 00:00:00.000000000Z
+date: 2014-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bio-logger
-  requirement: &23241080 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *23241080
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &23240480 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.8.0
+        version: '0'
   type: :development
   prerelease: false
-  version_requirements: *23240480
-- !ruby/object:Gem::Dependency
-  name: rdoc
-  requirement: &23239900 !ruby/object:Gem::Requirement
-    none: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.12'
-  type: :development
-  prerelease: false
-  version_requirements: *23239900
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: cucumber
-  requirement: &23238540 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *23238540
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &23237420 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '0'
   type: :development
   prerelease: false
-  version_requirements: *23237420
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &23236880 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.3
+        version: 2.0.0
   type: :development
   prerelease: false
-  version_requirements: *23236880
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: bio
-  requirement: &23236300 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.4.2
   type: :development
   prerelease: false
-  version_requirements: *23236300
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.2
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &23014480 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.12'
+        version: '0'
   type: :development
   prerelease: false
-  version_requirements: *23014480
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: regressiontest
-  requirement: &23013780 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.2
   type: :development
   prerelease: false
-  version_requirements: *23013780
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.2
 description: Functions and tools for tranforming and changing tab delimited and comma
   separated table files - useful for Excel sheets and SQL/RDF output
 email: pjotr.public01@thebird.nl
@@ -120,9 +132,9 @@ extra_rdoc_files:
 - LICENSE.txt
 - README.md
 files:
-- .document
-- .rspec
-- .travis.yml
+- ".document"
+- ".rspec"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -148,6 +160,7 @@ files:
 - lib/bio-table/merge.rb
 - lib/bio-table/overlap.rb
 - lib/bio-table/parser.rb
+- lib/bio-table/parsers/fastareader.rb
 - lib/bio-table/rdf.rb
 - lib/bio-table/rewrite.rb
 - lib/bio-table/statistics.rb
@@ -160,6 +173,7 @@ files:
 - lib/bio-table/validator.rb
 - spec/bio-table_spec.rb
 - spec/spec_helper.rb
+- test/data/input/aa.fa
 - test/data/input/table1.csv
 - test/data/input/table2.csv
 - test/data/input/table_no_headers.txt
@@ -186,29 +200,25 @@ files:
 homepage: http://github.com/pjotrp/bioruby-table
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 2239873243896303303
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Swiss army knife of tabulated data; transforming/filtering tab/csv files
 test_files: []