bio-gadget 0.2.3 → 0.2.4

data/bio-gadget.gemspec

@@ -18,4 +18,5 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'parallel'
   gem.add_dependency 'levenshtein-ffi'
   gem.add_dependency 'bio-faster'
+  gem.add_dependency 'mkfifo'
 end

data/lib/bio-gadget.rb

@@ -5,6 +5,8 @@ require 'bio-gadget/fqxz'
 require 'bio-gadget/qvstat'
 require 'bio-gadget/wigchr'
 
+require 'tempfile'
+
 module Bio
   class Gadget < Thor
 
@@ -22,5 +24,12 @@ module Bio
       end
     end
 
+    def mytempfile(basename, tmpdir = Dir::tmpdir)
+      fp = Tempfile.open(basename, tmpdir)
+      path = fp.path
+      fp.close #!
+      path
+    end
+
   end
 end

data/lib/bio-gadget/demlt.rb

@@ -1,17 +1,18 @@
 require 'bio-faster'
 require 'levenshtein'
+require 'mkfifo'
 require 'parallel'
-require 'thread'
 
 module Bio
   class Gadget < Thor
     namespace :bio
 
-    desc 'demlt BC POS', 'demultiplex fastq (via STDIN) by barcodes'
-    option 'output-dir', :type => :string, :default  => '.', :aliases => '-o'
-    def demlt(bcfile, tmpofs)
+    desc 'demlt BC POS LEN', 'demultiplex fastq (via STDIN) by barcodes'
+    option 'output-dir', :aliases => '-o', :type => :string, :default  => '.'
+    def demlt(bcfile, tmpofs, tmplen)
 
       ofs = tmpofs.to_i
+      len = tmplen.to_i
 
       wells = Array.new
       bcs = Array.new
@@ -25,62 +26,147 @@ module Bio
 
       bclens.uniq!
       if bclens.size != 1
-        raise 'Inconsistent barcode sequences'
+        raise 'Inconsistent barcode sequence lengths'
       end
       bclen = bclens[0]
 
-      ts = Array.new
-      qs = Array.new
-      (wells + ['other']).each { |well|
-        q = Queue.new
-        t = Thread.new(well, q) do |well, q|
-          tc = Thread.current
-          tc[:file] = "#{options['output-dir']}/#{well}.fq.xz"
-          fp = open("| xz -z -c -e > #{tc[:file]}", 'w')
-          tc[:read] = 0
-          while vals = q.shift
-            if vals == ""
-              break
-            else
-              fp.puts(vals)
-              tc[:read] = tc[:read] + 1
-            end
-          end
-          fp.close()
+      procs = Parallel.processor_count
+
+      fifo1paths = Array.new
+      procs.times { |i| fifo1paths.push(mytempfile('fifo1-')) }
+      pid = Kernel.fork {
+        fifo1s = Array.new
+        fifo1paths.each { |fifo1path| fifo1s.push(open(fifo1path, 'w+')) }
+        total = 0
+        Bio::Faster.new(:stdin).each_record(:quality => :raw) do |vals|
+          fifo1 = fifo1s[total % procs]
+          fifo1.puts(vals.join("\t"))
+          fifo1.flush
+          total += 1
+        end
+        fifo1s.each { |fifo1| fifo1.puts('*'); fifo1.close }
+        Kernel.exit!
+      }
+
+      fifo1paths.each { |fifo1path|
+        until File.exist?(fifo1path)
+          sleep 1
         end
-        qs.push(q)
-        ts.push(t)
       }
 
-      Bio::Faster.new(:stdin).each_record(:quality => :raw) do |seqid, seq, qvs|
-        tmpdists = Hash.new
-        bcs.each_index { |bcidx|
-          tmpdists[bcidx] = Levenshtein.distance(bcs[bcidx], seq[ofs, bclen])
+      fifo2paths = Array.new
+      procs.times { |i|
+        fifo2path = mytempfile('fifo2-')
+        fifo2paths.push(fifo2path)
+        pid = Kernel.fork {
+          open(fifo2path, 'w+') { |fifo2|
+            fifo1 = open(fifo1paths[i], 'r+')
+            while true
+              line = fifo1.gets
+              if line.nil?
+                sleep 1
+              elsif line == "*\n"
+                break
+              else
+                seqid, seq, qvs = line.rstrip.split(/\t/)
+                tmpdists = Hash.new
+                bcs.each_index { |bcidx|
+                  tmpdists[bcidx] = Levenshtein.distance(bcs[bcidx], seq[ofs, bclen])
+                }
+                dists = tmpdists.sort { |a, b| a[1] <=> b[1] }
+                bc = dists[0][1] < 2 && dists[0][1] < dists[1][1] ? dists[0][0] : -1
+                fifo2.puts("#{bc}\t#{seqid}\t#{seq}\t#{qvs}")
+                fifo2.flush
+              end
+            end
+            fifo1.close
+            fifo2.puts('*')
+          }
+          Kernel.exit!
         }
-        dists = tmpdists.sort { |a, b| a[1] <=> b[1] }
-        if dists[0][1] < 2 && dists[0][1] < dists[1][1]
-          qs[dists[0][0]].push("@#{seqid}\n#{seq}\n+\n#{qvs}")
-        else
-          qs[-1].push("@#{seqid}\n#{seq}\n+\n#{qvs}")
+      }
+
+      fifo2paths.each { |fifo2path|
+        until File.exist?(fifo2path)
+          sleep 1
         end
-        Thread.pass
-      end
+      }
 
-      qs.each { |q| q.push('') }
-      ts.each { |t| t.join }
+      tmpwells = wells + ['other']
 
-      total = 0
-      bcs.each_index { |i|
-        t = ts[i]
-        r = t[:read]
-        puts "#{bcs[i]}\t#{r}\t#{t[:file]}"
-        total = total+r
+      fifo3paths = Array.new
+      tmpwells.each_index { |i| fifo3paths.push(mytempfile('fifo3-')) }
+      pid = Kernel.fork {
+        fifo2s = Array.new
+        fifo2paths.each { |fifo2path| fifo2s.push(open(fifo2path, 'r+')) }
+        fifo2done = Hash.new
+        fifo3s = Array.new
+        fifo3paths.each { |fifo3path| fifo3s.push(open(fifo3path, 'w+')) }
+        fifo2s.cycle { |fifo2|
+          unless fifo2done.key?(fifo2)
+            line = fifo2.gets
+            if line.nil?
+              sleep 1
+            elsif line == "*\n"
+              # puts("#{fifo2} eof.")
+              fifo2done[fifo2] = ''
+            else
+              bcs, seqid, seq, qvs = line.rstrip.split(/\t/)
+              fifo3 = fifo3s[bcs.to_i]
+              fifo3.puts([seqid, seq, qvs].join("\t"))
+              fifo3.flush
+            end
+          end
+          if fifo2done.size == fifo2s.size
+            break
+          end
+        }
+        fifo2s.each { |fifo2| fifo2.close }
+        fifo3s.each { |fifo3| fifo3.puts('*'); fifo3.close }
+        Kernel.exit!
+      }
+
+      fifo3paths.each { |fifo3path|
+        until File.exist?(fifo3path)
+          sleep 1
+        end
+      }
+
+      tmpwells.each_index { |i|
+        well = tmpwells[i]
+        outpath = "#{options['output-dir']}/#{well}.fq.xz"
+        pid = Kernel.fork {
+          left = ofs+bclen
+          right = ofs+bclen+len-1
+          preprocess = ofs > 0 ? <<"DEDUPandFORMAT"
+| ruby -F'\\t' -anle 'f1=$F[1][0..#{right}];f2=$F[2][0..#{right}];puts([f1+f2, $F[0], f2, f1].join("\\t"))' \\
+| sort -k 1 -r | cut -f 2- | uniq -f 2 \\
+| ruby -F'\\t' -anle 'puts(["@"+$F[0], $F[2][#{left}..-1], "+", $F[1][#{left}..-1]].join("\\n"))' \\
+DEDUPandFORMAT
+          : <<"FORMAT"
+| ruby -F'\\t' -anle 'puts(["@"+$F[0], $F[1][#{left}..-1], "+", $F[2][#{left}..-1].rstrip].join("\\n"))' \\
+FORMAT
+          preprocess += "| xz -z -c -e > #{outpath}"
+          open(preprocess, 'w') { |fp|
+            fifo3 = open(fifo3paths[i], 'r+')
+            while true
+              line = fifo3.gets
+              if line.nil?
+                sleep 1
+              elsif line == "*\n"
+                break
+              else
+                fp.puts(line)
+                fp.flush
+              end
+            end
+            fifo3.close
+          }
+          Kernel.exit!
+        }
       }
-      t = ts[-1]
-      r = t[:read]
-      puts "Other\t#{r}\t#{t[:file]}"
-      puts '===='
-      puts "Total\t#{total+r}"
+
+      Process.waitall
 
     end
 

data/lib/bio-gadget/version.rb

@@ -3,7 +3,7 @@ require 'thor'
 module Bio
   class Gadget < Thor
 
-    VERSION = "0.2.3"
+    VERSION = "0.2.4"
 
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bio-gadget
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-21 00:00:00.000000000 Z
+date: 2013-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gthor
@@ -75,6 +75,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mkfifo
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Gadgets for bioinformatics
 email:
 - shintaro.katayama@gmail.com