scbi_multi_gz_reader 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bff6f483b2c9eb929aecd04e2054cadbfcd38407
+  data.tar.gz: f4822e17db2570421cabd7a1859eb28c8d9217cd
+SHA512:
+  metadata.gz: 8b64117d3461220161b04fb5fbfbccaafeb24719fa7861753447992eff3c077f6a390b55406f89dbed13dfe856cbbaf2167eb74969dec12732fe680ed444969a
+  data.tar.gz: 2b4b01f1d4bda02b04bc943aa84668d79217fb7b7a88372b53bf4c9cbbccf21d24546bc6e844b890c39c0614c6e333b56d9991f424f191bdd5fd59a158956054

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in scbi_multi_gz_reader.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 dariogf
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,58 @@
+# ScbiMultiGzReader
+
+Zlib::GzipReader is not able to read multiple stream gz/gzip files, instead it only reads the first stream and then closes the file without warnings so you only get a part of the file when you expected to get the full one.
+
+This is a wrapper to read gz/gzip files  composed of multiple streams (eg, made by doing a cat to join some gz files in a bigger one.
+
+By now it only has the readline and eof? instance methods, the ones needed by scbi_fastq to read fastq files.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'scbi_multi_gz_reader'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install scbi_multi_gz_reader
+
+## Usage
+
+	# read checking for eof
+
+	file=MultiGzReader.new('file.gz'))
+    
+    while !file.eof? do
+      puts file.readline
+    end 
+
+    file.close
+
+    ------
+
+    # read checking for res.nil?
+	file=MultiGzReader.new('file.gz'))
+    
+	loop do
+      res=file.readline
+      break if res.nil?
+      puts res
+    end
+
+    file.close
+
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/scbi_multi_gz_reader/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.pattern = "test/*_test.rb"
+end

data/lib/scbi_multi_gz_reader/multi_gz_reader.rb

@@ -0,0 +1,51 @@
+require 'zlib'
+
+class MultiGzReader
+
+	def initialize(file_name)
+		@file = File.open(file_name)
+		@io = Zlib::GzipReader.new @file
+	end
+
+	def readline
+
+		res=nil
+
+		begin
+			res=@io.readline
+		rescue EOFError => e
+
+			#reached END, check if there is more data to read
+			unused = @io.unused
+
+			@io.finish
+
+			# there is something left to read, open another stream
+			if !unused.nil?
+				#puts "FIN1, fpos: #{@file.pos}, unused: #{unused.length}, io_eof: #{@io.eof}, eof: #{@file.eof}"
+			
+				@file.pos -= unused.length
+				@io = Zlib::GzipReader.new @file
+				#repeat the read so there is no eof error
+				res=readline
+
+			else
+				#no more data to read, return nil
+				res=nil
+			end
+		end
+
+		return res
+	end
+
+	def eof?
+		#nothing more to read
+		@io.unused.nil? && (@io.closed? || @io.eof?) && (@file.closed? || @file.eof?)
+	end
+
+	def close
+		#@io.finish
+		@file.close
+	end
+
+end

data/lib/scbi_multi_gz_reader/version.rb

@@ -0,0 +1,3 @@
+module ScbiMultiGzReader
+  VERSION = "0.0.1"
+end

data/lib/scbi_multi_gz_reader.rb

@@ -0,0 +1,6 @@
+require "scbi_multi_gz_reader/version"
+require "scbi_multi_gz_reader/multi_gz_reader"
+
+module ScbiMultiGzReader
+  # Your code goes here...
+end

data/scbi_multi_gz_reader.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'scbi_multi_gz_reader/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "scbi_multi_gz_reader"
+  spec.version       = ScbiMultiGzReader::VERSION
+  spec.authors       = ["dariogf"]
+  spec.email         = ["dariogf@scbi.uma.es"]
+  spec.summary       = %q{Reads gz/gzip files with multiple streams in ruby}
+  spec.description   = %q{Wrapper to read gz/gzip files  composed of multiple gz streams (eg, made by doing a cat to join some gz files in a bigger one.)}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/test/fq_all2std.pl

@@ -0,0 +1,194 @@
+#!/usr/local/bin/perl -w
+
+# Author: lh3
+
+use strict;
+use warnings;
+use Getopt::Std;
+
+my $usage = qq(
+Usage:   fq_all2std.pl <command> <in.txt>
+
+Command: scarf2std      Convert SCARF format to the standard/Sanger FASTQ
+         fqint2std      Convert FASTQ-int format to the standard/Sanger FASTQ
+         sol2std        Convert Solexa/Illumina FASTQ to the standard FASTQ
+         fa2std         Convert FASTA to the standard FASTQ
+         fq2fa          Convert various FASTQ-like format to FASTA
+         instruction    Explanation to different format
+         example        Show examples of various formats
+
+Note:    Read/quality sequences MUST be presented in one line.
+\n);
+
+die($usage) if (@ARGV < 1);
+
+# Solexa->Sanger quality conversion table
+my @conv_table;
+for (-64..64) {
+  $conv_table[$_+64] = chr(int(33 + 10*log(1+10**($_/10.0))/log(10)+.499));
+}
+
+# parsing command line
+my $cmd = shift;
+my %cmd_hash = (scarf2std=>\&scarf2std, fqint2std=>\&fqint2std, sol2std=>\&sol2std, fa2std=>\&fa2std,
+				fq2fa=>\&fq2fa, example=>\&example, instruction=>\&instruction);
+if (defined($cmd_hash{$cmd})) {
+  &{$cmd_hash{$cmd}};
+} else {
+  die("** Unrecognized command $cmd");
+}
+
+sub fa2std {
+  my %opts = (q=>25);
+  getopts('q:', \%opts);
+  my $q = chr($opts{q} + 33);
+  warn("-- The default quality is set to $opts{q}. Use '-q' at the command line to change the default.\n");
+  while (<>) {
+	if (/^>(\S+)/) {
+	  print "\@$1\n";
+	  $_ = <>;
+	  print "$_+\n", $q x (length($_)-1), "\n";
+	}
+  }
+}
+
+sub fq2fa {
+  while (<>) {
+	if (/^@(\S+)/) {
+	  print ">$1\n";
+	  $_ = <>; print;
+	  <>; <>;
+	}
+  }
+}
+
+sub scarf2std {
+  while (<>) {
+	my @t = split(':', $_);
+	my $name = join('_', @t[0..4]);
+	print "\@$name\n$t[5]\n+\n";
+	my $qual = '';
+	@t = split(/\s/, $t[6]);
+	$qual .= $conv_table[$_+64] for (@t);
+	print "$qual\n";
+  }
+}
+
+sub fqint2std {
+  while (<>) {
+	if (/^@/) {
+	  print;
+	  $_ = <>; print; $_ = <>; $_ = <>;
+	  my @t = split;
+	  my $qual = '';
+	  $qual .= $conv_table[$_+64] for (@t);
+	  print "+\n$qual\n";
+	}
+  }
+}
+
+sub sol2std {
+  my $max = 0;
+  while (<>) {
+	if (/^@/) {
+	  print;
+	  $_ = <>; print; $_ = <>; $_ = <>;
+	  my @t = split('', $_);
+	  my $qual = '';
+	  $qual .= $conv_table[ord($_)] for (@t);
+	  print "+\n$qual\n";
+	}
+  }
+}
+
+sub instruction {
+
+  print "
+FASTQ format is first used in the Sanger Institute, and therefore
+we take the Sanger specification as the standard FASTQ. Although
+Solexa/Illumina reads file looks pretty much like the standard
+FASTQ, they are different in that the qualities are scaled
+differently. In the quality string, if you can see a character
+with its ASCII code higher than 90, probably your file is in the
+Solexa/Illumina format.
+
+Sometimes we also use an integer, instead of a single character,
+to explicitly show the qualities. In that case, negative
+qualities indicates that Solexa/Illumina qualities are used.
+
+";
+
+}
+
+sub example {
+  my $exam_scarf = '
+USI-EAS50_1:4:2:710:120:GTCAAAGTAATAATAGGAGATTTGAGCTATTT:23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 19 23 23 23 18 23 23 23
+USI-EAS50_1:4:2:690:87:GTTTTTTTTTTTCTTTCCATTAATTTCCCTTT:23 23 23 23 23 23 23 23 23 23 23 23 12 23 23 23 23 23 16 23 23 9 18 23 23 23 12 23 18 23 23 23
+USI-EAS50_1:4:2:709:32:GAGAAGTCAAACCTGTGTTAGAAATTTTATAC:23 23 23 23 23 23 23 23 20 23 23 23 23 23 23 23 23 23 23 23 23 12 23 18 23 23 23 23 23 23 23 23
+USI-EAS50_1:4:2:886:890:GCTTATTTAAAAATTTACTTGGGGTTGTCTTT:23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23
+USI-EAS50_1:4:2:682:91:GGGTTTCTAGACTAAAGGGATTTAACAAGTTT:23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 20 23 23 23 23 23 23 23 23 23 23 23 18 23 23 23 23
+USI-EAS50_1:4:2:663:928:GAATTTGTTTGAAGAGTGTCATGGTCAGATCT:23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23
+';
+
+  my $exam_fqint = '
+@4_1_912_360
+AAGGGGCTAGAGAAACACGTAATGAAGGGAGGACTC
++4_1_912_360
+40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 21 40 40 40 40 40 40 40 40 40 26 40 40 14 39 40 40
+@4_1_54_483
+TAATAAATGTGCTTCCTTGATGCATGTGCTATGATT
++4_1_54_483
+40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 16 40 40 40 28 40 40 40 40 40 40 16 40 40 5 40 40
+@4_1_537_334
+ATTGATGATGCTGTGCACCTAGCAAGAAGTTGCATA
++4_1_537_334
+40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 21 29 40 40 33 40 40 33 40 40 33 31 40 40 40 40 18 26 40 -2
+@4_1_920_361
+AACGGCACAATCCAGGTTGATGCCTACGGCGGGTAC
++4_1_920_361
+40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 39 40 40 40 40 40 40 40 40 31 40 40 40 40 40 40 15 5 -1 3
+@4_1_784_155
+AATGCATGCTTCGAATGGCATTCTCTTCAATCACGA
++4_1_784_155
+40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 31 40 40 40 40 40
+@4_1_595_150
+AAAGACGTGGCCAGATGGGTGGCCAAGTGCCCGACT
++4_1_595_150
+40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 30 40 40 40 40 40 40 40 40 40 20 40 40 40 40 40 14 40 40
+';
+
+  my $exam_sol = '
+@SLXA-B3_649_FC8437_R1_1_1_610_79
+GATGTGCAATACCTTTGTAGAGGAA
++SLXA-B3_649_FC8437_R1_1_1_610_79
+YYYYYYYYYYYYYYYYYYWYWYYSU
+@SLXA-B3_649_FC8437_R1_1_1_397_389
+GGTTTGAGAAAGAGAAATGAGATAA
++SLXA-B3_649_FC8437_R1_1_1_397_389
+YYYYYYYYYWYYYYWWYYYWYWYWW
+@SLXA-B3_649_FC8437_R1_1_1_850_123
+GAGGGTGTTGATCATGATGATGGCG
++SLXA-B3_649_FC8437_R1_1_1_850_123
+YYYYYYYYYYYYYWYYWYYSYYYSY
+@SLXA-B3_649_FC8437_R1_1_1_362_549
+GGAAACAAAGTTTTTCTCAACATAG
++SLXA-B3_649_FC8437_R1_1_1_362_549
+YYYYYYYYYYYYYYYYYYWWWWYWY
+@SLXA-B3_649_FC8437_R1_1_1_183_714
+GTATTATTTAATGGCATACACTCAA
++SLXA-B3_649_FC8437_R1_1_1_183_714
+YYYYYYYYYYWYYYYWYWWUWWWQQ
+';
+
+  print qq(
+solexa
+======
+$exam_sol
+scarf
+=====
+$exam_scarf
+fqint
+=====
+$exam_fqint
+);
+}

data/test/gz.rb

@@ -0,0 +1,11 @@
+require '../lib/scbi_fastq/multi_gz_reader'
+
+ file=MultiGzReader.new(File.join(File.dirname(__FILE__),'minitest.fastq.gz'))
+
+    loop do
+      res=file.readline
+      puts "LINE: #{res}"
+      break if res.nil?
+    end
+
+    file.close

data/test/scbi_multi_gz_reader_test.rb

@@ -0,0 +1,113 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class ScbiMultiGzReaderTest < Test::Unit::TestCase
+
+  def setup
+
+  end
+
+  def test_mini_multi_gz_reader
+
+    # test a file with multiple gz streams
+
+    file=MultiGzReader.new(File.join(File.dirname(__FILE__),'minitest.fastq.gz'))
+    
+    i=0
+
+    loop do
+      res=file.readline
+      #puts res
+      break if res.nil?
+      i = i+1
+    end
+
+    file.close
+
+    assert_equal(28,i)
+
+  end
+
+
+  def test_mini_multi_gz_reader_EOF_while
+
+    # test a file with multiple gz streams
+
+    file=MultiGzReader.new(File.join(File.dirname(__FILE__),'minitest.fastq.gz'))
+    
+    i=0
+
+    begin
+      res=file.readline
+      #puts res
+      i = i+1
+    end while !file.eof?
+
+    file.close
+
+    assert_equal(28,i)
+
+  end
+
+  def test_mini_multi_gz_reader_EOF_until
+
+    # test a file with multiple gz streams
+
+    file=MultiGzReader.new(File.join(File.dirname(__FILE__),'minitest.fastq.gz'))
+    
+    i=0
+
+    begin
+      res=file.readline
+      #puts res
+      i = i+1
+    end until file.eof?
+
+    file.close
+
+    assert_equal(28,i)
+
+  end
+
+def test_empty_multi_gz_reader_EOF_until
+
+    # test a file with multiple gz streams
+
+    file=MultiGzReader.new(File.join(File.dirname(__FILE__),'empty.fastq.gz'))
+    
+    i=0
+
+    while !file.eof? do
+      res=file.readline
+      #puts res
+      i = i+1
+    end 
+
+    file.close
+
+    assert_equal(0,i)
+
+  end
+
+
+  def test_mini_complete_gz
+    #test only one gz stream in file
+
+    file=MultiGzReader.new(File.join(File.dirname(__FILE__),'minitest_full.fastq.gz'))
+    
+    i=0
+
+    loop do
+      res=file.readline
+      #puts res
+      break if res.nil?
+      i = i+1
+    end
+
+    file.close
+
+    assert_equal(i,28)
+
+  end
+
+  
+end

data/test/test_helper.rb

@@ -0,0 +1,3 @@
+require 'stringio'
+require 'test/unit'
+require File.dirname(__FILE__) + '/../lib/scbi_multi_gz_reader'

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: scbi_multi_gz_reader
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- dariogf
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-11-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Wrapper to read gz/gzip files  composed of multiple gz streams (eg, made
+  by doing a cat to join some gz files in a bigger one.)
+email:
+- dariogf@scbi.uma.es
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/scbi_multi_gz_reader.rb
+- lib/scbi_multi_gz_reader/multi_gz_reader.rb
+- lib/scbi_multi_gz_reader/version.rb
+- scbi_multi_gz_reader.gemspec
+- test/empty.fastq.gz
+- test/fq_all2std.pl
+- test/gz.rb
+- test/h1.fastq.gz
+- test/h2.fastq.gz
+- test/minitest.fastq.gz
+- test/minitest_full.fastq.gz
+- test/scbi_multi_gz_reader_test.rb
+- test/test_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Reads gz/gzip files with multiple streams in ruby
+test_files:
+- test/empty.fastq.gz
+- test/fq_all2std.pl
+- test/gz.rb
+- test/h1.fastq.gz
+- test/h2.fastq.gz
+- test/minitest.fastq.gz
+- test/minitest_full.fastq.gz
+- test/scbi_multi_gz_reader_test.rb
+- test/test_helper.rb