scbi_fqbin 0.2.1

data/History.txt

@@ -0,0 +1,11 @@
+=== 0.0.1 2010-05-14
+
+* 1 major enhancement:
+  * Initial release
+  
+ === 0.1.0 2010-05-31
+
+* 1 major enhancement:
+  * Added extras to sequences
+  
+  

data/Manifest.txt

@@ -0,0 +1,12 @@
+History.txt
+Manifest.txt
+PostInstall.txt
+README.rdoc
+Rakefile
+script/console
+script/destroy
+script/generate
+lib/scbi_fqbin/fbin_file.rb
+lib/scbi_fqbin.rb
+test/test_helper.rb
+test/test_scbi_fbin_file.rb

data/PostInstall.txt

@@ -0,0 +1,7 @@
+
+For more information on scbi_fqbin, see http://scbi_fqbin.rubyforge.org
+
+NOTE: Change this information in PostInstall.txt 
+You can also delete it if you don't want it.
+
+

data/README.rdoc

@@ -0,0 +1,79 @@
+= scbi_fqbin
+
+* http://github.com/dariogf/scbi_fqbin
+
+== DESCRIPTION:
+
+scbi_fqbin is a gem to handle compressed fastq and fasta files (FQbin files) made by Rafael Larrosa, Gonzalo Claros & Dario Guerrero for the Plataforma Andaluza de Bioinformática.
+
+FastaBin was developed to handle huge fasta and qual files in a more efficient manner. To achieve its goals, FastaBin uses some 
+indexing and compression techniques in order to minimize disk accesses and thus increase the performance when reading fasta and qual files.
+
+== FEATURES/PROBLEMS:
+
+
+== SYNOPSIS:
+
+=== Using command line tools:
+
+* To generate a FastaBin file:
+
+	mk_fbin fasta qual extras fbin_file
+
+* To read a random sequence fomr FastaBin file:
+
+	rd_seq_fbin fbin_file seqname
+	
+* To iterate over all sequences in FastaBin file:
+	
+
+=== Using ruby FastaBin gem:
+
+
+== REQUIREMENTS:
+
+*libfbin library
+
+To install libfbin library, download this file http://www.scbi.uma.es/downloads/gems/libfbin.zip:
+
+then unzip with:
+
+unzip libfbin.zip
+
+and install with:
+
+make
+
+sudo make install
+
+By default the library will be installed to /usr/local/lib and binaries (mk_fbin, rd_seq_fbin and iterate_fbin) to /usr/local/bin
+
+== INSTALL:
+
+gem install scbi_fqbin
+
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2010 Rafael Larrosa, Gonzalo Claros & Dario Guerrero
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,28 @@
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+require 'fileutils'
+# require './lib/scbi_fqbin'
+
+Hoe.plugin :newgem
+Hoe.plugin :website
+# Hoe.plugin :cucumberfeatures
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.spec 'scbi_fqbin' do
+  self.developer 'Dario Guerrero', 'dariogf@gmail.com'
+  self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
+  self.rubyforge_name       = self.name # TODO this is default value
+  self.extra_deps         = [['scbi_fasta','>= 0.0.1']]
+
+end
+
+require 'newgem/tasks'
+Dir['tasks/**/*.rake'].each { |t| load t; puts t}
+
+# TODO - want other tests/tasks run by default? Add them to the list
+
+# remove_task :default
+task :default => [:build, :install,:spec, :features]
+

data/lib/scbi_fqbin/fbin_file.rb

@@ -0,0 +1,223 @@
+require 'ffi'
+
+class FFIString<FFI::MemoryPointer
+  
+  def initialize
+    super(:pointer,1)
+    # self.write_string('a')      
+    # return FFI::MemoryPointer.from_string('a'*150000000)
+  end
+  
+  def to_s
+    resPtr = self.read_pointer()
+
+    #if null, return nil, if not return string
+    return resPtr.null? ? nil : resPtr.read_string()
+  end
+  
+  def inspect
+     self.to_s
+  end
+  
+end
+
+class FbinFile
+  
+  CREATE_NEW_FILE=1
+  APPEND_TO_FILE=2
+  extend FFI::Library
+  
+  ffi_lib(["libfbin"])
+
+    functions = [
+
+      [:read_seq, [:string,:string,:pointer,:pointer,:pointer],:int],
+      [:read_data_sequential, [:pointer,:pointer,:pointer,:pointer,:pointer],:int],
+      [:initialize_sequential_reads,[:pointer, :string],:int],
+      [:close_sequential_reads,[:pointer],:int],
+      [:write_seq,[:pointer, :string, :string, :string, :string],:int],
+      [:close_writes,[:pointer],:int],
+      [:initialize_writes,[:pointer, :string, :int, :int, :int, :int],:int],
+      [:inspect_file_data_struct,[:pointer],:void],
+      [:free_string,[:pointer],:int]
+      
+      
+    ]
+
+    functions.each do |func|
+      begin
+        attach_function(*func)
+        private func[0]
+      rescue Object => e
+        puts "Could not attach #{func}, #{e.message}"
+      end
+    end
+  
+  
+  # Initializes the file 
+  def initialize(file_path, mode = 'r', qual_to_phred=true, qual_to_array=true, discretize_qual=0, flatten_qual=0, create_index=1)
+    @file_path = file_path
+    @open_mode=mode
+    
+    @qual_to_phred = qual_to_phred
+    @qual_to_array = qual_to_array
+    
+    @to_phred = lambda{|q| q - 33}
+    
+    if @open_mode.index('r')
+      @gzf_bin = FFI::MemoryPointer.new :pointer
+      initialize_sequential_reads(@gzf_bin,@file_path)
+      # inspect_file_data_struct(@gzf_bin)
+      @gzf_bin = @gzf_bin.get_pointer(0)
+      
+    elsif @open_mode.index('w')
+      @write_struct = FFI::MemoryPointer.new :pointer
+      initialize_writes(@write_struct,@file_path, CREATE_NEW_FILE,discretize_qual, flatten_qual, create_index)
+      @write_struct = @write_struct.get_pointer(0)
+      # inspect_file_data_struct(@write_struct)
+      
+    elsif @open_mode.index('a')
+       
+       @write_struct = FFI::MemoryPointer.new :pointer
+       initialize_writes(@write_struct,@file_path, APPEND_TO_FILE,discretize_qual, flatten_qual, create_index)
+       @write_struct = @write_struct.get_pointer(0)
+       # inspect_file_data_struct(@write_struct)
+       
+    else
+      raise "Invalid aperture mode #{mode}. Use r/w"
+    end
+  end
+  
+  # Access a sequence by its name using the indexed read
+  def read_sequence(seq_name)
+    
+    fastaPtr = FFIString.new
+    qualPtr = FFIString.new
+    extrasPtr = FFIString.new
+        
+    if read_seq(@file_path,seq_name,fastaPtr,qualPtr,extrasPtr)==0
+      
+      qual=qualPtr.to_s
+      
+      if @qual_to_phred
+        qual=qual.each_char.map{|e| (@to_phred.call(e.ord))}
+        if !@qual_to_array
+          qual=qual.join(' ')
+        end
+      end
+      
+      fasta=fastaPtr.to_s
+      extras = extrasPtr.to_s
+      
+      free_string(qualPtr)
+      free_string(fastaPtr)
+      free_string(extrasPtr)
+
+      return seq_name, fasta, qual, extras
+      
+    else
+      
+      free_string(qualPtr)
+      free_string(fastaPtr)
+      free_string(extrasPtr)
+      
+      return nil
+      # raise "Invalid sequence"
+    end
+  end
+  
+  # Returns the next sequence in file, or nil if no more ara available
+  def next_sequence
+    return get_next_seq(@gzf_bin)
+  end
+  
+  def get_next_seq(gzf_bin)
+    seq_namePtr = FFIString.new
+    fastaPtr = FFIString.new
+    qualPtr = FFIString.new
+    extrasPtr = FFIString.new
+    
+    if ((r=read_data_sequential(gzf_bin, seq_namePtr, fastaPtr, qualPtr, extrasPtr))==0)
+      
+      qual=qualPtr.to_s
+      if @qual_to_phred
+        qual=qual.each_char.map{|e| (@to_phred.call(e.ord))}
+        
+        if !@qual_to_array
+          qual=qual.join(' ')
+        end
+      end
+      
+      seq_name=seq_namePtr.to_s
+      fasta=fastaPtr.to_s
+      extras=extrasPtr.to_s
+      
+      # seq_namePtr.free
+      # fastaPtr.free
+      # qualPtr.free
+      # extrasPtr.free
+      
+      free_string(seq_namePtr)
+      free_string(fastaPtr)
+      free_string(qualPtr)
+      free_string(extrasPtr)
+      
+      return seq_name, fasta, qual, extras
+      
+    else
+      free_string(seq_namePtr)
+      free_string(fastaPtr)
+      free_string(qualPtr)
+      free_string(extrasPtr)
+      return nil
+    end
+  end
+  
+  # Iterates over all sequences in file
+  def each
+                                                   
+    gzf_bin = FFI::MemoryPointer.new :pointer
+    initialize_sequential_reads(gzf_bin,@file_path)
+    gzf_bin = gzf_bin.get_pointer(0)
+    
+     while seq=get_next_seq(gzf_bin)
+       yield seq
+     end
+     
+    close_sequential_reads(gzf_bin)
+  end
+  
+  # Writes a sequence to file
+  def write_sequence(seq_name,fasta,qual,extras)
+
+    if qual.is_a?(Array)
+        qual=qual.join(' ')
+    end
+      
+    write_seq(@write_struct,seq_name,fasta,qual,extras)
+  end
+  
+  
+  # Closes file
+  def close
+
+    if @open_mode.index('r')
+      close_sequential_reads(@gzf_bin)
+    else # CREATE_NEW_FILE or APPEND_TO_FILE
+      # inspect_write_struct(@write_struct)
+      close_writes(@write_struct)
+    end
+    
+  end
+  
+  # Returns the number of sequences in file. Iterating over it
+  def count
+    res=0
+    each do #|n,f,q,e|
+      # puts n
+      res+=1
+    end
+    return res
+  end
+
+end

data/lib/scbi_fqbin.rb

@@ -0,0 +1,8 @@
+$: << File.join(File.dirname(__FILE__),File.basename(__FILE__,File.extname(__FILE__)))
+  
+# require 'fastabin'
+require 'fbin_file'
+
+module ScbiFastabin
+  VERSION = '0.2.1'
+end

data/script/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs =  " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs <<  " -r #{File.dirname(__FILE__) + '/../lib/scbi_fqbin.rb'}"
+puts "Loading scbi_fqbin gem"
+exec "#{irb} #{libs} --simple-prompt"

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/test/test_helper.rb

@@ -0,0 +1,3 @@
+require 'stringio'
+require 'test/unit'
+require File.dirname(__FILE__) + '/../lib/scbi_fqbin'

data/test/test_scbi_fbin_file.rb

@@ -0,0 +1,158 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+# Ojo si va muy lento cuando se incrementa el numero de secuencias porque resulta que las longitudes de la secuencia va aumentando con cada número
+
+class TestScbiFbinFile < Test::Unit::TestCase
+
+  def setup
+  	#File.delete(TEST_FILE)
+  end
+  
+	  TEST_FILE='/tmp/fbinfile';
+  	
+    SEQ_FASTA='ACTG'
+  	SEQ_FASTA400=SEQ_FASTA*100
+		SEQ_QUAL='B'
+		SEQ_QUAL400=(SEQ_QUAL*400*SEQ_FASTA.length)
+		PHRED_QUAL=[33]
+	  SEQ_NAME='SEQ'
+	  SEQ_EXTRAS='SOME EXTRAS IN SEQ'
+	  
+	  PROGRESS=100
+ 
+  def this_method
+     caller[0] =~ /`([^']*)'/ and $1
+  end
+
+  def get_seq(i)
+    # i=25
+    return SEQ_FASTA*i
+  end
+  
+  def get_qual(i)
+    # i=25
+    return (SEQ_QUAL*i*SEQ_FASTA.length)#.join(' ')
+  end
+ 
+  def fill_file(n)
+   fb=FbinFile.new(TEST_FILE,'w',false)
+   n.times do |c|
+     i = c+1
+     if (i%PROGRESS)==0
+        puts "#{this_method}: #{i}"
+     end
+     
+     fb.write_sequence(SEQ_NAME+i.to_s,get_seq(i),get_qual(i),SEQ_EXTRAS)
+     #fb.write_sequence(SEQ_NAME+i.to_s,SEQ_FASTA400,SEQ_QUAL400,SEQ_EXTRAS)
+   end
+   puts "END #{this_method}"
+   fb.close
+  end
+  
+  def test_new
+  
+    if File.exists?(TEST_FILE)
+       File.delete(TEST_FILE)
+    end
+      
+    fb=FbinFile.new(TEST_FILE,'w',false)
+    # fb.write_sequence('hola','actg','50 50 50 50','extras')
+    fb.close
+    
+    assert(File.exists?(TEST_FILE))
+  end
+   
+  def test_add100
+    num_seqs=100
+    # make new file and fill with data
+    fill_file(num_seqs)
+  
+    fb=FbinFile.new(TEST_FILE,'r',false)
+    
+    assert_equal(num_seqs,fb.count)
+    fb.close
+  
+  end
+     
+  def test_read_random
+    num_seqs=100 #100
+         # make new file and fill with data
+    fill_file(100)
+  
+  
+    fb=FbinFile.new(TEST_FILE,'r',false)
+  
+    num_seqs.times do |c|
+      i = c+1
+      n,s,q,e=fb.read_sequence(SEQ_NAME+i.to_s)
+      # puts n,s,q,s.length,q.length,'_______'
+      assert_equal(SEQ_NAME+i.to_s,n)
+      assert_equal(get_seq(i),s)
+      assert_equal(get_qual(i), q)
+      assert_equal(SEQ_EXTRAS , e)
+      # gets
+    end
+   
+    fb.close      
+  end
+  
+  def test_read_no_exists
+  
+     fill_file(100)
+   
+     fb=FbinFile.new(TEST_FILE,'r',false)
+     n,s,q=fb.read_sequence(SEQ_NAME+'NO_EXIST')
+     puts "H:#{n}"
+     assert(n.nil?)
+     fb.close
+  end
+  
+  
+   
+   def test_each
+       fill_file(100)
+    
+       i = 1
+       fb=FbinFile.new(TEST_FILE,'r',false)
+       
+       fb.each do |n,s,q,e|
+         assert_equal(SEQ_NAME+i.to_s,n)
+         assert_equal(get_seq(i),s)
+         assert_equal(get_qual(i),q)
+         assert_equal(SEQ_EXTRAS,e)
+         
+         i+=1
+       end       
+   
+       assert_equal(i,101)
+   
+       fb.close     
+   
+   end
+   
+  def test_each_by_index
+    num_seqs=100
+    
+    fill_file(num_seqs)
+  
+    i = 1
+    
+    fb=FbinFile.new(TEST_FILE,'r',false)
+  
+    fb.each do |n,f,q,e|
+      if (i%PROGRESS)==0
+         puts "#{this_method}: #{i}"
+      end
+      assert_equal(SEQ_NAME+i.to_s,n)
+      assert_equal(get_seq(i),f)
+      assert_equal(get_qual(i),q)
+      assert_equal(SEQ_EXTRAS,e)
+      i+=1
+    end
+  
+    assert_equal(i,num_seqs+1)
+    fb.close
+  
+  end           
+   
+end

metadata

@@ -0,0 +1,137 @@
+--- !ruby/object:Gem::Specification
+name: scbi_fqbin
+version: !ruby/object:Gem::Version
+  version: 0.2.1
+  prerelease: 
+platform: ruby
+authors:
+- Dario Guerrero
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: scbi_fasta
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: newgem
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.3
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.6'
+description: ! "scbi_fqbin is a gem to handle compressed fastq and fasta files (FQbin
+  files) made by Rafael Larrosa, Gonzalo Claros & Dario Guerrero for the Plataforma
+  Andaluza de Bioinformática.\n\nFastaBin was developed to handle huge fasta and qual
+  files in a more efficient manner. To achieve its goals, FastaBin uses some \nindexing
+  and compression techniques in order to minimize disk accesses and thus increase
+  the performance when reading fasta and qual files."
+email:
+- dariogf@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- History.txt
+- Manifest.txt
+- PostInstall.txt
+- README.rdoc
+files:
+- History.txt
+- Manifest.txt
+- PostInstall.txt
+- README.rdoc
+- Rakefile
+- script/console
+- script/destroy
+- script/generate
+- lib/scbi_fqbin/fbin_file.rb
+- lib/scbi_fqbin.rb
+- test/test_helper.rb
+- test/test_scbi_fbin_file.rb
+- .gemtest
+homepage: http://github.com/dariogf/scbi_fqbin
+licenses: []
+post_install_message: PostInstall.txt
+rdoc_options:
+- --main
+- README.rdoc
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: scbi_fqbin
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: scbi_fqbin is a gem to handle compressed fastq and fasta files (FQbin files)
+  made by Rafael Larrosa, Gonzalo Claros & Dario Guerrero for the Plataforma Andaluza
+  de Bioinformática
+test_files:
+- test/test_helper.rb
+- test/test_scbi_fbin_file.rb