bio-gadget 0.1.4 → 0.2.1

data/README.org

@@ -17,8 +17,9 @@ Currently available commands are
 
 : bio
 : ---
+: gthor bio:dedup          # deduplicate fastq (via STDIN)
 : gthor bio:demlt BC POS   # demultiplex fastq (via STDIN) by barcodes
-: gthor bio:fqlzma         # automatic (re)compression of *.fq(.gz|.bz2) files
+: gthor bio:fqxz           # automatic (re)compression of *.fq(.gz|.bz2) files
 : gthor bio:qvstat QUAL    # statistics of quality values in *.qual file
 : gthor bio:wigchr WIG CHR # extract wiggle track on specified chromosome
 

data/lib/bio-gadget.rb

@@ -1,6 +1,7 @@
 require 'bio-gadget/version'
+require 'bio-gadget/dedup'
 require 'bio-gadget/demlt'
-require 'bio-gadget/fqlzma'
+require 'bio-gadget/fqxz'
 require 'bio-gadget/qvstat'
 require 'bio-gadget/wigchr'
 

data/lib/bio-gadget/dedup.rb

@@ -0,0 +1,33 @@
+require 'bio-faster'
+require 'parallel'
+
+module Bio
+  class Gadget < Thor
+    namespace :bio
+
+    desc 'dedup', 'deduplicate fastq (via STDIN)'
+    def dedup
+
+      p1in, p1out = IO.pipe
+
+      fork {
+        p1in.close
+        $stdout.reopen(p1out)
+        open("| sort -k 1 -r -S #{sprintf('%2d', 100/(Parallel.processor_count+1))}% -T $TMPDIR | cut -f 2- | uniq -f 2", 'w') { |fp|
+          Bio::Faster.new(:stdin).each_record(:quality => :raw) do |seqid, seq, qvs|
+            fp.puts "#{seq}#{qvs}\t#{seqid}\t#{qvs}\t#{seq}"
+          end
+        }
+      }
+
+      p1out.close
+
+      p1in.each_line { |line|
+        seqid, qvs, seq = line.rstrip.split
+        puts "@#{seqid}\n#{seq}\n+\n#{qvs}"
+      }
+
+    end
+
+  end
+end

data/lib/bio-gadget/demlt.rb

@@ -8,7 +8,7 @@ module Bio
     namespace :bio
 
     desc 'demlt BC POS', 'demultiplex fastq (via STDIN) by barcodes'
-    option :destdir, :type => :string, :default  => '.'
+    option 'output-dir', :type => :string, :default  => '.', :aliases => '-o'
     def demlt(bcfile, tmpofs)
 
       ofs = tmpofs.to_i
@@ -35,8 +35,8 @@ module Bio
         q = SizedQueue.new(100000)
         t = Thread.new(well, q) do |well, q|
           tc = Thread.current
-          tc[:file] = "#{options[:destdir]}/#{well}.fq.gz"
-          fp = open("| gzip -c > #{tc[:file]}", 'w')
+          tc[:file] = "#{options['output-dir']}/#{well}.fq.xz"
+          fp = open("| xz -z -c -e > #{tc[:file]}", 'w')
           tc[:read] = 0
           while vals = q.shift
             if vals == ""
@@ -52,19 +52,28 @@ module Bio
         ts.push(t)
       }
 
+      rq = Queue.new
+      Thread.new(rq) {
+        Bio::Faster.new(:stdin).each_record(:quality => :raw) do |seqid, seq, qvs|
+          rq.push([seqid, seq, qvs])
+        end
+        rq.push('')
+      }
+
       seqs = Array.new
-      Bio::Faster.new(:stdin).each_record(:quality => :raw) do |seqid, seq, qvs|
-        seqs.push([seqid, seq, qvs])
-        if seqs.size == 100000 * Parallel.processor_count
+      while vals = rq.shift
+        if vals != ""
+          seqs.push(vals)
+        end
+        if vals == "" || seqs.size == 100000 * Parallel.processor_count
           parallel_Levenshtein(seqs, bcs, ofs, bclen, qs)
           seqs = Array.new
         end
+        if vals == ""
+          qs.each { |q| q.push('') }
+          break
+        end
       end
-      if seqs.size > 0
-        parallel_Levenshtein(seqs, bcs, ofs, bclen, qs)
-      end
-
-      qs.each { |q| q.push('') }
       ts.each { |t| t.join }
 
       total = 0
@@ -104,9 +113,9 @@ module Bio
         dists = tmpdist.sort { |a, b| a[1] <=> b[1] }
         if dists[0][1] < dists[1][1] && dists[0][1] < 2
           idx = dists[0][0]
-          qs[idx].push(">#{seqid}\n#{seq}\n+\n#{qvs}")
+          qs[idx].push("@#{seqid}\n#{seq}\n+\n#{qvs}")
         else
-          qs[-1].push(">#{seqid}\n#{seq}\n+\n#{qvs}")
+          qs[-1].push("@#{seqid}\n#{seq}\n+\n#{qvs}")
         end
       end
 

data/lib/bio-gadget/{fqlzma.rb → fqxz.rb}

@@ -6,11 +6,11 @@ module Bio
 
     namespace :bio
 
-    desc 'fqlzma', 'automatic (re)compression of *.fq(.gz|.bz2) files'
-    def fqlzma
+    desc 'fqxz', 'automatic (re)compression of *.fq(.gz|.bz2) files'
+    def fqxz
       Parallel.map(Pathname.glob('*.fq{.gz,.bz2,}')) { |fqfilename|
-        lzmafilename = fqfilename.sub(/\.fq(\.(gz|bz2))*$/, '.fq.lzma')
-        if !lzmafilename.exist?
+        xzfilename = fqfilename.sub(/\.fq(\.(gz|bz2))*$/, '.fq.xz')
+        if !xzfilename.exist?
           case fqfilename.extname
           when '.gz'
             decompressor = 'gunzip -c'
@@ -19,9 +19,9 @@ module Bio
           else
             decompressor = 'cat'
           end
-          puts "compressing #{lzmafilename}..."
-          system "#{decompressor} #{fqfilename} | lzma -c > #{lzmafilename} 2> #{lzmafilename}.log"
-          system "lzma -t #{lzmafilename} >> #{lzmafilename}.log 2>&1"
+          puts "compressing #{xzfilename}..."
+          system "#{decompressor} #{fqfilename} | xz -z -e -c > #{xzfilename} 2> #{xzfilename}.log"
+          system "xz -t #{xzfilename} >> #{xzfilename}.log 2>&1"
         end
       }
     end

data/lib/bio-gadget/version.rb

@@ -3,7 +3,7 @@ require 'thor'
 module Bio
   class Gadget < Thor
 
-    VERSION = "0.1.4"
+    VERSION = "0.2.1"
 
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bio-gadget
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-07 00:00:00.000000000 Z
+date: 2012-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gthor
@@ -90,8 +90,9 @@ files:
 - Rakefile
 - bio-gadget.gemspec
 - lib/bio-gadget.rb
+- lib/bio-gadget/dedup.rb
 - lib/bio-gadget/demlt.rb
-- lib/bio-gadget/fqlzma.rb
+- lib/bio-gadget/fqxz.rb
 - lib/bio-gadget/qvstat.rb
 - lib/bio-gadget/version.rb
 - lib/bio-gadget/wigchr.rb