scbi_mapreduce 0.0.29

data/skeleton/remove_mids/lib/global_match.rb

@@ -0,0 +1,97 @@
+class GMatch
+  
+  attr_accessor :offset
+  attr_accessor :match
+  
+  
+end
+
+class String
+		def lcs(s2)
+			s1=self
+			res="" 
+			num=Array.new(s1.size){Array.new(s2.size)}
+			len,ans=0
+			lastsub=0
+			s1.scan(/./).each_with_index do |l1,i |
+				s2.scan(/./).each_with_index do |l2,j |
+				  unless l1==l2
+				    num[i][j]=0
+				  else
+				    (i==0 || j==0)? num[i][j]=1 : num[i][j]=1 + num[i-1][j-1]
+				    if num[i][j] > len
+				      len = ans = num[i][j]
+				      thissub = i
+				      thissub -= num[i-1][j-1] unless num[i-1][j-1].nil?  
+				      if lastsub==thissub
+				        res+=s1[i,1]
+				      else
+				        lastsub=thissub
+				        res=s1[lastsub, (i+1)-lastsub]
+				      end
+				    end
+				  end
+				end
+			end
+			res
+		end
+end
+
+
+class Regexp
+  def global_match(input_str,overlap_group_no = 0)
+    res = []
+    
+    str=input_str
+    
+    last_end = 0
+    
+    loop do
+      str = input_str.slice(last_end,input_str.length-last_end)
+      if str.nil? or str.empty?
+        break
+      end
+      
+      m = self.match(str)
+      # puts "find in: #{str}"
+      
+      if !m.nil?
+        # puts m.inspect
+        
+        
+        new_match=GMatch.new()
+        new_match.offset = last_end
+        new_match.match = m
+        
+        res.push new_match
+        
+        if overlap_group_no == 0
+          last_end += m.end(overlap_group_no)
+        else
+          last_end += m.begin(overlap_group_no)
+        end
+        
+      else
+        break
+      end
+      
+    end
+    
+    
+    return res
+  end
+    
+  
+  # def global_match(str, &proc)
+  #     retval = nil
+  #     loop do
+  #       res = str.sub(self) do |m|
+  #         proc.call($~) # pass MatchData obj
+  #         ''
+  #       end
+  #       break retval if res == str
+  #       str = res
+  #       retval ||= true
+  #     end
+  #   end
+end

data/skeleton/remove_mids/linear_implementation.rb

@@ -0,0 +1,87 @@
+#!/usr/bin/env ruby
+
+# load required libraries
+require 'scbi_mapreduce'
+
+# in order to load fastq files
+require 'scbi_fastq'
+
+# modify include path
+$: << File.join(File.dirname(__FILE__),'lib')
+
+require 'find_mids'
+include FindMids
+
+# check arguments
+if ARGV.count != 2
+
+  puts "Usage #{File.basename($0)} fastq_file chunk"
+  puts ""
+  puts "#{File.basename($0)} iterates over all sequences in fastq_file (a file in FastQ format) and removes a KNOWN_MID from it"
+  exit
+end
+
+fastq_file_path=ARGV[0]
+
+if !File.exists?(fastq_file_path)
+  puts "Error, #{fastq_file_path} doesn't exists"
+  exit
+end
+
+
+# make processing
+
+# open files
+@@fastq_file=FastqFile.new(fastq_file_path)
+@@results=FastqFile.new('./results2.fastq','w+')
+
+# process
+chunk_size=ARGV[1].to_i
+
+# iterate over file
+begin
+  seqs=[]
+
+  chunk_size.times do
+    # read data from file
+    name,fasta,qual,comments=@@fastq_file.next_seq
+    
+    if name.nil?
+      break
+    end
+    seqs<<[name,fasta,qual,comments]
+
+  end
+
+  if !seqs.empty?
+
+
+    # process it
+    find_mid_without_blast(seqs)
+
+    # # find a known MID position
+    # pos=fasta.upcase.index(KNOWN_MID)
+    #
+    # if pos
+    #
+    #   # keep fasta from pos to end
+    #   fasta.slice!(0,pos + KNOWN_MID.length)
+    #   # keep qual from pos to end
+    #   qual.slice!(0,pos + KNOWN_MID.length)
+    #
+    # end
+    #
+
+    # write data to disk
+    seqs.each do |name,fasta,qual,comments|
+        @@results.write_seq(name,fasta,qual,comments)
+    end
+
+  end
+
+end until seqs.empty?
+
+
+# close files
+@@fastq_file.close
+@@results.close

data/skeleton/remove_mids/main.rb

@@ -0,0 +1,89 @@
+#!/usr/bin/env ruby
+
+$: << File.dirname(__FILE__)
+
+# load required libraries
+require 'scbi_mapreduce'
+
+# in order to load fastq files
+require 'scbi_fastq'
+
+require 'my_worker_manager.rb'
+
+# check arguments
+if ARGV.count != 3
+  
+  puts "Usage #{File.basename($0)} fastq_file workers chunk_size"
+  puts ""
+  puts "#{File.basename($0)} iterates over all sequences in fastq_file (a file in FastQ format) and removes a MID (barcode) from it"  
+  exit
+end
+
+fastq_file_path=ARGV.shift
+
+if !File.exists?(fastq_file_path)
+  puts "Error, #{fastq_file_path} doesn't exists"
+  exit
+end 
+
+# listen on all ips at port 50000
+server_ip='10.243'
+ip_list = Socket.ip_address_list.select{|e| e.ipv4?}.map{|e| e.ip_address}
+
+ip=ip_list.select{|ip| ip.index(server_ip)==0}.first
+
+if !ip
+  ip='0.0.0.0'
+end
+
+  port = 0
+
+# set number of workers. You can also provide an array with worker names.
+# Those workers names can be read from a file produced by the existing
+# queue system, if any.
+
+
+
+workers = 4
+
+# read optional workers parameter
+  input_workers = ARGV.shift
+if !input_workers.nil?
+  # if it is a file
+  if File.exists?(input_workers)
+    # read workers into array
+    workers=File.read(input_workers).split("\n").map{|w| w.chomp}
+  else
+    # workers is a number
+    workers = input_workers.to_i
+  end
+end
+
+chunk_size = ARGV.shift.to_i
+
+# we need the path to my_worker in order to launch it when necessary
+custom_worker_file = File.join(File.dirname(__FILE__),'my_worker.rb')
+
+# initialize the work manager. Here you can pass parameters like file names
+MyWorkerManager.init_work_manager(fastq_file_path)
+
+# launch processor server
+mgr = ScbiMapreduce::Manager.new(ip, port, workers, MyWorkerManager, custom_worker_file, STDOUT,'~seqtrimnext/init_env')
+
+# you can set additional properties
+# =================================
+
+# if you want basic checkpointing. Some performance drop should be expected
+# mgr.checkpointing=true
+
+# if you want to keep the order of input data. Some performance drop should be expected
+# mgr.keep_order=true
+
+# you can set the size of packets of data sent to workers
+mgr.chunk_size=chunk_size
+
+# start processing
+mgr.start_server
+
+# this line is reached when all data has been processed
+puts "Program finished"

data/skeleton/remove_mids/my_worker.rb

@@ -0,0 +1,59 @@
+
+# adjust import paths
+$: << File.join(File.dirname(__FILE__),'lib')
+
+# load external module
+require 'find_mids'
+include FindMids
+
+
+# MyWorker defines the behaviour of workers.
+# Here is where the real processing takes place
+class MyWorker < ScbiMapreduce::Worker
+
+  # starting_worker method is called one time at initialization
+  # and allows you to initialize your variables
+  def starting_worker
+
+    # You can use worker logs at any time in this way:
+    # $WORKER_LOG.info "Starting a worker"
+
+  end
+
+
+  # receive_initial_config is called only once just after
+  # the first connection, when initial parameters are
+  # received from manager
+  def receive_initial_config(parameters)
+
+    # Reads the parameters
+
+    # You can use worker logs at any time in this way:
+    # $WORKER_LOG.info "Params received"
+
+    # save received parameters, if any
+    # @params = parameters
+  end
+
+
+  # process_object method is called for each received object.
+  # Be aware that objs is always an array, and you must iterate
+  # over it if you need to process it independently
+  #
+  # The value returned here will be received by the work_received
+  # method at your worker_manager subclass.
+  def process_object(objs)
+
+    # find mid in sequences
+    # find_mid_with_blast(objs)
+    find_mid_without_blast(objs)
+
+    # return modified objs back to manager
+    return objs
+  end
+
+
+  def closing_worker
+
+  end
+end

data/skeleton/remove_mids/my_worker_manager.rb

@@ -0,0 +1,68 @@
+require 'json'
+
+# MyWorkerManager class is used to implement the methods
+# to send and receive the data to or from workers
+class MyWorkerManager < ScbiMapreduce::WorkManager
+
+  # init_work_manager is executed at the start, prior to any processing.
+  # You can use init_work_manager to initialize global variables, open files, etc...
+  # Note that an instance of MyWorkerManager will be created for each
+  # worker connection, and thus, all global variables here should be
+  # class variables (starting with @@)
+  def self.init_work_manager(fastq_file_path)
+
+    # puts "tiempo1",Time.now
+    # open file using scbi_fastq gem
+    @@fastq_file=FastqFile.new(fastq_file_path)
+    @@results=FastqFile.new('./results.fastq'+Time.now.usec.to_s,'w+')
+    @@cache = []
+    
+    # @@fastq_file.each do |name,fasta,qual,comments|
+    #   @@cache << [name,fasta,qual,comments]
+    # end
+    # puts "tiempo2",Time.now
+  end
+
+  # end_work_manager is executed at the end, when all the process is done.
+  # You can use it to close files opened in init_work_manager
+  def self.end_work_manager
+    @@fastq_file.close
+
+    @@results.close
+  end
+
+  # worker_initial_config is used to send initial parameters to workers.
+  # The method is executed once per each worker
+  def worker_initial_config
+    
+  end
+
+  # next_work method is called every time a worker needs a new work
+  # Here you can read data from disk
+  # This method must return the work data or nil if no more data is available
+  def next_work
+    name,fasta,qual,comments=@@fastq_file.next_seq
+    # name,fasta,qual,comments=@@cache.shift
+
+    if !name.nil?
+      return name,fasta,qual,comments
+    else
+      return nil
+    end
+
+  end
+
+
+  # work_received is executed each time a worker has finished a job.
+  # Here you can write results down to disk, perform some aggregated statistics, etc...
+  def work_received(results)
+
+    # write results to disk
+    results.each do |name,fasta,qual,comments|
+      # puts "comments: #{comments}\n"
+      @@results.write_seq(name,fasta,qual,comments)
+    end
+    
+  end
+
+end

data/skeleton/simple/README.txt

@@ -0,0 +1,16 @@
+A simple scbi_mapreduce application demo
+========================================
+
+This application is only a basic template. You should modify the files
+to perform the desired tasks. There are other templates available, you 
+can list them by issuing this command:
+
+scbi_mapreduce
+
+You can launch the application right now with the following command:
+
+  ruby main.rb
+
+A server and some workers will be launched, and 200.000 1Kb strings will be 
+converted from downcase to uppercase, in blocks of 100 strings.
+

data/skeleton/simple/main.rb

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+
+# load required libraries
+require 'scbi_mapreduce'
+require './my_worker_manager.rb'
+
+# listen on all ips at port 50000
+ip='0.0.0.0'
+port = 50000
+
+# set number of workers. You can also provide an array with worker names.
+# Those workers names can be read from a file produced by the existing
+# queue system, if any.
+workers = 8
+
+# we need the path to my_worker in order to launch it when necessary
+custom_worker_file = File.join(File.dirname(__FILE__),'my_worker.rb')
+
+# initialize the work manager. Here you can pass parameters like file names
+MyWorkerManager.init_work_manager
+
+# launch processor server
+mgr = ScbiMapreduce::Manager.new(ip, port, workers, MyWorkerManager, custom_worker_file, STDOUT)
+
+# you can set additional properties
+# =================================
+
+# if you want basic checkpointing. Some performance drop should be expected
+# mgr.checkpointing=true
+
+# if you want to keep the order of input data. Some performance drop should be expected
+# mgr.keep_order=true
+
+# you can set the size of packets of data sent to workers
+mgr.chunk_size=100
+
+# start processing
+mgr.start_server
+
+# this line is reached when all data has been processed
+puts "Program finished"