bio-faster 0.2.2 → 0.4.2

data/.travis.yml

@@ -3,10 +3,9 @@ rvm:
   - 1.8.7
   - 1.9.2
   - 1.9.3
- # - jruby-18mode # JRuby in 1.8 mode
- # - jruby-19mode # JRuby in 1.9 mode
+  - jruby-18mode # JRuby in 1.8 mode
+  - jruby-19mode # JRuby in 1.9 mode
  # - rbx-18mode
  # - rbx-19mode
 # uncomment this line if your project needs to run something other than `rake`:
-#script: bundle exec rake ext:build
 #script: bundle exec rake

data/Gemfile

@@ -2,6 +2,7 @@ source "http://rubygems.org"
 # Add dependencies required to use your gem here.
 # Example:
 #   gem "activesupport", ">= 2.3.5"
+gem "ffi"
 
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
@@ -12,4 +13,5 @@ group :development do
   gem "rcov", ">= 0"
   gem "bio", ">= 1.4.2"
   gem "rspec"
+  gem "ffi"
 end

data/Gemfile.lock

@@ -3,6 +3,7 @@ GEM
   specs:
     bio (1.4.2)
     diff-lcs (1.1.3)
+    ffi (1.0.11)
     git (1.2.5)
     jeweler (1.6.4)
       bundler (~> 1.0)
@@ -28,6 +29,7 @@ PLATFORMS
 DEPENDENCIES
   bio (>= 1.4.2)
   bundler (~> 1.0.0)
+  ffi
   jeweler (~> 1.6.4)
   rcov
   rspec

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2011 Francesco Strozzi
+Copyright (c) 2012 Francesco Strozzi
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -3,8 +3,7 @@
 Bio::Faster
 ==========
 
-Fast and simple parser for FastA / FastQ files, based on Heng Li Kseq library written in C.
-http://lh3lh3.users.sourceforge.net/parsefastq.shtml
+Fast and simple parser for FastQ files
 
 Examples
 ========
@@ -28,6 +27,6 @@ Contributing to bio-faster
 Copyright
 =========
 
-Copyright (c) 2011 Francesco Strozzi. See LICENSE.txt for
+Copyright (c) 2012 Francesco Strozzi. See LICENSE.txt for
 further details.
 

data/Rakefile

@@ -17,11 +17,11 @@ Jeweler::Tasks.new do |gem|
   gem.name = "bio-faster"
   gem.homepage = "http://github.com/fstrozzi/bioruby-faster"
   gem.license = "MIT"
-  gem.summary = %Q{A fast parser for Fasta and FastQ files}
-  gem.description = %Q{A fast parser for Fasta and FastQ files}
+  gem.summary = %Q{A fast parser for FastQ files}
+  gem.description = %Q{A fast parser for FastQ files}
   gem.email = "francesco.strozzi@gmail.com"
   gem.authors = ["Francesco Strozzi"]
-  gem.required_ruby_version = '>= 1.9'
+  gem.files << "lib/*/**"
   # dependencies defined in Gemfile
 end
 Jeweler::RubygemsDotOrgTasks.new
@@ -58,20 +58,14 @@ Rake::RDocTask.new do |rdoc|
 end
 
 namespace :ext do
-  desc "Compile extension"
+  desc "Build native extension"
   task :build do
-    puts "Building extension"
-    cd File.join(File.dirname(__FILE__),"ext")
-    sh "ruby "+File.join(File.dirname(__FILE__),"ext","extconf.rb")
-    sh "make"
-    FileList["*.log"].each do |file|
-      rm file
-    end
-    FileList["*.o"].each do |file|
-      rm file
-    end
-    cd ".."  
+    cd "ext"
+    ruby "mkrf_conf.rb"
+    sh "rake"
+    cd ".."
   end
+
 end
 
 task :default => ["ext:build",:spec]

data/VERSION

@@ -1 +1 @@
-0.2.2
+0.4.2

data/bio-faster.gemspec

@@ -5,14 +5,14 @@
 
 Gem::Specification.new do |s|
   s.name = "bio-faster"
-  s.version = "0.2.2"
+  s.version = "0.4.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Francesco Strozzi"]
-  s.date = "2012-04-02"
-  s.description = "A fast parser for Fasta and FastQ files"
+  s.date = "2012-04-27"
+  s.description = "A fast parser for FastQ files"
   s.email = "francesco.strozzi@gmail.com"
-  s.extensions = ["ext/extconf.rb"]
+  s.extensions = ["ext/mkrf_conf.rb"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.md"
@@ -27,49 +27,93 @@ Gem::Specification.new do |s|
     "Rakefile",
     "VERSION",
     "bio-faster.gemspec",
-    "ext/extconf.rb",
     "ext/faster.c",
-    "ext/kseq.h",
+    "ext/mkrf_conf.rb",
     "lib/bio-faster.rb",
+    "lib/bio/faster.rb",
+    "lib/bio/faster/library.rb",
+    "spec/fastq_error_spec.rb",
     "spec/helper.rb",
     "spec/parser_spec.rb",
-    "test/data/sample.fasta",
-    "test/data/sample.fastq",
-    "test/data/sample.fastq.gz",
-    "test/data/sff_sample.fastq"
+    "test/data/errors/error_header.fastq",
+    "test/data/errors/error_long_qual.fastq",
+    "test/data/errors/error_qual_del.fastq",
+    "test/data/errors/error_qual_escape.fastq",
+    "test/data/errors/error_qual_null.fastq",
+    "test/data/errors/error_qual_space.fastq",
+    "test/data/errors/error_qual_tab.fastq",
+    "test/data/errors/error_qual_unit_sep.fastq",
+    "test/data/errors/error_qual_vtab.fastq",
+    "test/data/errors/error_spaces.fastq",
+    "test/data/errors/error_tabs.fastq",
+    "test/data/errors/error_trunc_at_qual.fastq",
+    "test/data/errors/error_trunc_at_seq.fastq",
+    "test/data/errors/error_trunc_in_qual.fastq",
+    "test/data/errors/error_trunc_in_seq.fastq",
+    "test/data/formats/illumina_full_range_as_illumina.fastq",
+    "test/data/formats/illumina_full_range_as_sanger.fastq",
+    "test/data/formats/illumina_full_range_as_solexa.fastq",
+    "test/data/formats/illumina_full_range_original_illumina.fastq",
+    "test/data/formats/longreads_as_illumina.fastq",
+    "test/data/formats/longreads_as_sanger.fastq",
+    "test/data/formats/longreads_as_solexa.fastq",
+    "test/data/formats/misc_dna_as_illumina.fastq",
+    "test/data/formats/misc_dna_as_sanger.fastq",
+    "test/data/formats/misc_dna_as_solexa.fastq",
+    "test/data/formats/misc_dna_original_sanger.fastq",
+    "test/data/formats/misc_rna_as_illumina.fastq",
+    "test/data/formats/misc_rna_as_sanger.fastq",
+    "test/data/formats/misc_rna_as_solexa.fastq",
+    "test/data/formats/misc_rna_original_sanger.fastq",
+    "test/data/formats/sanger_full_range_as_illumina.fastq",
+    "test/data/formats/sanger_full_range_as_sanger.fastq",
+    "test/data/formats/sanger_full_range_as_solexa.fastq",
+    "test/data/formats/sanger_full_range_original_sanger.fastq",
+    "test/data/formats/solexa_full_range_as_illumina.fastq",
+    "test/data/formats/solexa_full_range_as_sanger.fastq",
+    "test/data/formats/solexa_full_range_as_solexa.fastq",
+    "test/data/formats/solexa_full_range_original_solexa.fastq",
+    "test/data/formats/wrapping_as_illumina.fastq",
+    "test/data/formats/wrapping_as_sanger.fastq",
+    "test/data/formats/wrapping_as_solexa.fastq"
   ]
   s.homepage = "http://github.com/fstrozzi/bioruby-faster"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.required_ruby_version = Gem::Requirement.new(">= 1.9")
   s.rubygems_version = "1.8.15"
-  s.summary = "A fast parser for Fasta and FastQ files"
+  s.summary = "A fast parser for FastQ files"
 
   if s.respond_to? :specification_version then
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<ffi>, [">= 0"])
       s.add_development_dependency(%q<shoulda>, [">= 0"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
       s.add_development_dependency(%q<bio>, [">= 1.4.2"])
       s.add_development_dependency(%q<rspec>, [">= 0"])
+      s.add_development_dependency(%q<ffi>, [">= 0"])
     else
+      s.add_dependency(%q<ffi>, [">= 0"])
       s.add_dependency(%q<shoulda>, [">= 0"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
       s.add_dependency(%q<rcov>, [">= 0"])
       s.add_dependency(%q<bio>, [">= 1.4.2"])
       s.add_dependency(%q<rspec>, [">= 0"])
+      s.add_dependency(%q<ffi>, [">= 0"])
     end
   else
+    s.add_dependency(%q<ffi>, [">= 0"])
     s.add_dependency(%q<shoulda>, [">= 0"])
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
     s.add_dependency(%q<rcov>, [">= 0"])
     s.add_dependency(%q<bio>, [">= 1.4.2"])
     s.add_dependency(%q<rspec>, [">= 0"])
+    s.add_dependency(%q<ffi>, [">= 0"])
   end
 end
 

data/ext/faster.c

@@ -1,57 +1,120 @@
+
 /*
-Copyright(C) 2011 Francesco Strozzi <francesco.strozzi@gmail.com>
-*/
+  Copyright(C) 2012 Francesco Strozzi <francesco.strozzi@gmail.com>
 
-#include <zlib.h>
+*/
 #include <stdio.h>
-#include "ruby.h"
-#include "kseq.h"
-
-KSEQ_INIT(gzFile, gzread)
-
-static VALUE method_parse(VALUE self, VALUE file) {
-
-	// check if a block is passed to the method
-	if (!(rb_block_given_p())) {
-		rb_raise(rb_eArgError,"You must pass a valid block!");
-	}
-
-	gzFile fp;
-	if (!(fp = gzopen(RSTRING_PTR(file), "r"))) {
-		rb_raise(rb_eArgError,"File %s not found!", RSTRING_PTR(file));
-	}
-	else {
-		kseq_t *seq;
-		seq = kseq_init(fp);
-		while (kseq_read(seq) >= 0) {
-			VALUE arr = rb_ary_new();
-			rb_ary_push(arr, rb_str_new2(seq->name.s));
-			if (seq->comment.l) {
-				rb_ary_push(arr, rb_str_new2(seq->comment.s));
-			}
-			else {
-				rb_ary_push(arr, Qnil);
-			}	
-			rb_ary_push(arr, rb_str_new2(seq->seq.s));
-			if (seq->qual.l) {
-				VALUE rb_quality = rb_ary_new();
-				int unsigned i = 0;
-				while(i < seq->qual.l) {
-					rb_ary_push(rb_quality,INT2FIX(*(seq->qual.s + i) - 33)); // quality conversion (Sanger/Phred only)
-					i++;
-				}
-				rb_ary_push(arr,rb_quality);
-			}
-			rb_yield(arr);
-		}
-		kseq_destroy(seq);
-		gzclose(fp);
-		return Qtrue;
-	}
+#include <string.h>
+#include <stdlib.h>
+
+#define _BSIZE 100000
+
+typedef struct {
+    char *id;
+    char *seq;
+    int *quality;
+    char *raw_quality;
+    char *filename;
+    char *line;
+    char *bad_chars;
+    FILE *stream;
+
+}FastQRecord;
+
+
+static char* alloc_and_copy(char *dst, char *src) {
+  if (dst==NULL || strlen(dst)<strlen(src)) {
+     if (dst!=NULL)
+        free(dst);
+     dst= malloc(sizeof (char)*(strlen(src)+1));
+  }
+  strcpy(dst, src);
+  int len;
+  len = strlen(dst);
+  if (dst[len-1] == '\n') dst[len-1] = '\0';
+  return dst;
+}
+
+
+const char* check_bad_chars(char *invalid_chars, char *string_to_check) {
+   return strpbrk(string_to_check, invalid_chars);
+}
+
+static char* initialize(char *ptr) {
+    if(ptr!=NULL){
+        free(ptr);
+        ptr = NULL;
+    }
+    return ptr;
 }
- 
-void Init_faster() {
-	VALUE Bio = rb_define_module("Bio");
-	VALUE Faster = rb_define_module_under(Bio,"Faster"); // it is defined as a sub-module of Bio
-	rb_define_singleton_method(Faster,"parse",method_parse,1);
+
+int* initialize_int(int *ptr) {
+    if(ptr!=NULL){
+        free(ptr);
+        ptr = NULL;
+    }
+    return ptr;
+}
+
+int check_header(char *header, char *firstline) {
+    if (*header == *firstline)
+        return 1;
+    else {
+        return 0;
+    }
 }
+
+int fastQ_iterator(FastQRecord *seq, int scale_factor) {
+    // initialization of structure elements.
+    char *header = "@"; // FastQ header
+    if (!seq->stream)
+      seq->stream = fopen(seq->filename,"r");
+    if (!seq->line)
+      seq->line = malloc(sizeof (char)* _BSIZE);
+    if (!seq->bad_chars)
+      seq->bad_chars = " \x1F\x7F\t\v\e";
+
+    // this is done to wipe out data from previous iteration
+    seq->id = initialize(seq->id);
+    seq->seq = initialize(seq->seq);
+    seq->raw_quality = initialize(seq->raw_quality);
+    for (int i = 0; i < 4; i++)
+    {
+      if (fgets(seq->line, _BSIZE, seq->stream) == NULL) {
+        // if either sequence or quality is missing the record is truncated
+        if((seq->seq != NULL && seq->raw_quality == NULL) || (seq->raw_quality != NULL && seq->seq == NULL)) return -2;
+        else return 0;
+      }
+      // getting seq ID
+      if (i==0) {
+        if (!check_header(header,seq->line)) return -1; // check if the header format is correct
+        // removing the @
+        seq->id = alloc_and_copy(seq->id, seq->line+1);
+
+      }
+      else {
+        if (check_bad_chars(seq->bad_chars,seq->line)) return -1; // check if quality or sequence includes bad characters
+        if (i==1) seq->seq = alloc_and_copy(seq->seq, seq->line);
+        if (i==3) {
+           seq->raw_quality = alloc_and_copy(seq->raw_quality, seq->line);
+           int quality_length = strlen(seq->raw_quality);
+           if(strlen(seq->seq) != strlen(seq->raw_quality)) return -2;  // if sequence and quality are of different length the record is truncated
+           int c = 0;
+           seq->quality = initialize_int(seq->quality);
+           seq->quality = malloc(sizeof (int)* quality_length);
+           while(c < quality_length) {
+                seq->quality[c] = *(seq->line + c) - scale_factor; // quality conversion
+            	  c++;
+           }
+
+        }
+
+      }
+    }
+
+    return 1;
+
+}
+
+
+#undef _BSIZE

data/ext/mkrf_conf.rb

@@ -0,0 +1,40 @@
+# create Rakefile for shared library compilation
+
+require File.join("..",File.dirname(__FILE__),"lib/bio/faster/library")
+
+path = File.expand_path(File.dirname(__FILE__))
+ext = Bio::Faster::Library.lib_extension
+
+flags = ""
+compile = ""
+if ext == "so" then
+  flags = "-shared -Wl,-soname,libfaster.so"
+  compile = " -fPIC"
+elsif ext == "dylib" then
+  flags = "-bundle -undefined dynamic_lookup -flat_namespace"
+end
+
+
+File.open(File.join(path,"Rakefile"),"w") do |rakefile|
+  rakefile.write <<-RAKE
+require 'rake/clean'
+
+source = %w(faster.c)
+
+CLEAN.include('*.o')
+SRC = FileList.new(source)
+OBJ_SRC = SRC.ext('o')
+
+rule '.o' => '.c' do |t|
+  sh "gcc#{compile} -std=c99 -c -g -Wall -O2 "+t.source+" -o "+t.name
+end
+
+task :compile_lib => OBJ_SRC do
+  sh "gcc #{flags} -std=c99 "+OBJ_SRC.join(" ")+" -o libfaster.#{ext}"
+end
+
+task :default => [:compile_lib, :clean]
+
+  RAKE
+
+end