scbi_mapreduce 0.0.29

data/skeleton/dummy_calcs/linear_implementation.rb

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+
+# load required libraries
+
+# modify include path
+$: << File.join(File.dirname(__FILE__),'lib')
+
+require 'calculations'
+include Calculations
+
+# process
+  
+times_to_calculate=1000
+
+times_to_calculate.times do
+
+  do_dummy_calculations
+
+end
+
+puts "final"
+

data/skeleton/dummy_calcs/main.rb

@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+
+$: << File.join(File.dirname(__FILE__))
+
+# load required libraries
+require 'scbi_mapreduce'
+require 'my_worker_manager'
+
+# listen on all ips at port 50000
+server_ip='10.243'
+ip_list = Socket.ip_address_list.select{|e| e.ipv4?}.map{|e| e.ip_address}
+
+ip=ip_list.select{|ip| ip.index(server_ip)==0}.first
+
+if !ip
+  ip='0.0.0.0'
+end
+
+port = 0
+
+# set number of workers. You can also provide an array with worker names.
+# Those workers names can be read from a file produced by the existing
+# queue system, if any.
+
+
+
+workers = 4
+
+# read optional workers parameter
+input_workers = ARGV.shift
+if !input_workers.nil?
+  # if it is a file
+  if File.exists?(input_workers)
+    # read workers into array
+    workers=File.read(input_workers).split("\n").map{|w| w.chomp}
+  else
+    # workers is a number
+    workers = input_workers.to_i
+  end
+end
+
+# we need the path to my_worker in order to launch it when necessary
+custom_worker_file = File.join(File.dirname(__FILE__),'my_worker.rb')
+
+# initialize the work manager. Here you can pass parameters like file names
+MyWorkerManager.init_work_manager
+
+# launch processor server
+mgr = ScbiMapreduce::Manager.new(ip, port, workers, MyWorkerManager, custom_worker_file, STDOUT)
+
+# you can set additional properties
+# =================================
+
+# if you want basic checkpointing. Some performance drop should be expected
+# mgr.checkpointing=true
+
+# if you want to keep the order of input data. Some performance drop should be expected
+# mgr.keep_order=true
+
+# you can set the size of packets of data sent to workers
+mgr.chunk_size=1
+
+# start processing
+mgr.start_server
+
+# this line is reached when all data has been processed
+puts "Program finished"

data/skeleton/dummy_calcs/my_worker.rb

@@ -0,0 +1,56 @@
+$: << File.join(File.dirname(__FILE__),'lib')
+
+require 'calculations'
+include Calculations
+
+# MyWorker defines the behaviour of workers.
+# Here is where the real processing takes place
+class MyWorker < ScbiMapreduce::Worker
+
+  # starting_worker method is called one time at initialization
+  # and allows you to initialize your variables
+  def starting_worker
+
+    # You can use worker logs at any time in this way:
+    # $WORKER_LOG.info "Starting a worker"
+
+  end
+
+
+  # receive_initial_config is called only once just after
+  # the first connection, when initial parameters are
+  # received from manager
+  def receive_initial_config(parameters)
+
+    # Reads the parameters
+
+    # You can use worker logs at any time in this way:
+    # $WORKER_LOG.info "Params received"
+
+    # save received parameters, if any
+    # @params = parameters
+  end
+
+
+  # process_object method is called for each received object.
+  # Be aware that objs is always an array, and you must iterate
+  # over it if you need to process it independently
+  #
+  # The value returned here will be received by the work_received
+  # method at your worker_manager subclass.
+  def process_object(objs)
+
+    # iterate over all objects received
+    # objs.each do |obj|
+      # convert to uppercase
+    do_dummy_calculations
+    # end
+    
+    # return objs back to manager
+    return objs
+  end
+
+  def closing_worker
+
+  end
+end

data/skeleton/dummy_calcs/my_worker_manager.rb

@@ -0,0 +1,52 @@
+require 'json'
+
+# MyWorkerManager class is used to implement the methods
+# to send and receive the data to or from workers
+class MyWorkerManager < ScbiMapreduce::WorkManager
+
+  # init_work_manager is executed at the start, prior to any processing.
+  # You can use init_work_manager to initialize global variables, open files, etc...
+  # Note that an instance of MyWorkerManager will be created for each
+  # worker connection, and thus, all global variables here should be
+  # class variables (starting with @@)
+  def self.init_work_manager
+    
+    # execute dummy_calc in workers @remaining_data times
+    @@remaining_data = 1000
+  end
+
+  # end_work_manager is executed at the end, when all the process is done.
+  # You can use it to close files opened in init_work_manager
+  def self.end_work_manager
+
+  end
+
+  # worker_initial_config is used to send initial parameters to workers.
+  # The method is executed once per each worker
+  def worker_initial_config
+
+  end
+
+  # next_work method is called every time a worker needs a new work
+  # Here you can read data from disk
+  # This method must return the work data or nil if no more data is available
+  def next_work
+    @@remaining_data -= 1
+
+    e = @@remaining_data
+
+    e = nil if @@remaining_data<=0
+    
+    return e
+
+  end
+
+
+  # work_received is executed each time a worker has finished a job.
+  # Here you can write results down to disk, perform some aggregated statistics, etc...
+  def work_received(results)
+
+    # write_data_to_disk(results)
+  end
+
+end

data/skeleton/dummy_calcs/threads_implementation.rb

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+
+# load required libraries
+
+# modify include path
+$: << File.join(File.dirname(__FILE__),'lib')
+
+require 'thread_pool'
+require 'calculations'
+include Calculations
+
+if ARGV.count!=1
+	puts "use: #{$0} threads" 
+	exit
+end
+
+@pool=ThreadPool.new(ARGV[0].to_i)
+
+# process
+  
+times_to_calculate=1000
+
+times_to_calculate.times do
+
+  @pool.process {do_dummy_calculations}
+
+end
+
+
+puts "wait"
+@pool.join
+puts "final"
+

data/skeleton/remove_mids/README.txt

@@ -0,0 +1,30 @@
+A simple scbi_mapreduce application demo
+========================================
+
+This application is a basic sequence processing template. It processes all 
+sequences in fastq_file (a file in FastQ format) removing a MIDs from it. It 
+needs some external requisites to work:
+
+EXTERNAL REQUISITES
+===================
+
+* scbi_blast gem installed
+
+
+At lib/db you can find a preformated MID database.
+
+You can modify the files to perform more complicated processing. 
+There are other templates available, you can list them by issuing this command:
+
+scbi_mapreduce
+
+You can launch the application right now with the following command using 4 cpus/cores and chunks of 100 sequences at a time:
+
+  ruby main.rb fastq_file 4 100
+
+A server and some workers will be launched, and all sequences in fastq_file will
+be processed in blocks of 100 sequences.
+
+A sequential lineal example is also provided, you can launch it by issuing:
+
+ruby linear_implementation.rb fastq_file

data/skeleton/remove_mids/launch_only_workers.rb

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+$: << File.dirname(__FILE__)
+
+require "logger"
+
+$: << '/Users/dariogf/progs/ruby/gems/scbi_mapreduce/lib'
+
+require 'scbi_mapreduce'
+require 'my_worker_manager'
+
+
+$LOG = Logger.new(STDOUT)
+$LOG.datetime_format = "%Y-%m-%d %H:%M:%S"
+
+ip='10.247.255.5'
+port = 50000
+workers = 8
+
+custom_worker_file = File.join(File.dirname(__FILE__),'my_worker.rb')
+
+$LOG.info 'Starting server'
+
+
+worker_launcher = ScbiMapreduce::WorkerLauncher.new(ip,port, workers, custom_worker_file, STDOUT)
+worker_launcher.launch_workers_and_wait
+
+# launch processor server
+$LOG.info 'Closing workers'

data/skeleton/remove_mids/lib/db/mids.fasta

@@ -0,0 +1,120 @@
+>RL1
+ACACGACGACT
+>RL2
+ACACGTAGTAT
+>RL3
+ACACTACTCGT
+>RL4
+ACGACACGTAT
+>RL5
+ACGAGTAGACT
+>RL6
+ACGCGTCTAGT
+>RL7
+ACGTACACACT
+>RL8
+ACGTACTGTGT
+>RL9
+ACGTAGATCGT
+>RL10
+ACTACGTCTCT
+>RL11
+ACTATACGAGT
+>RL12
+ACTCGCGTCGT
+>MID1
+ACGAGTGCGT
+>MID2
+ACGCTCGACA
+>MID3
+AGACGCACTC
+>MID5
+ATCAGACACG
+>MID6
+ATATCGCGAG
+>MID7
+CGTGTCTCTA
+>MID8
+CTCGCGTGTC
+>MID10
+TCTCTATGCG
+>MID11
+TGATACGTCT
+>MID13
+CATAGTAGTG
+>MID14
+CGAGAGATAC
+>MID15
+ATACGACGTA
+>MID16
+TCACGTACTA
+>MID17
+CGTCTAGTAC
+>MID18
+TCTACGTAGC
+>MID19
+TGTACTACTC
+>MID20
+ACGACTACAG
+>MID21
+CGTAGACTAG
+>MID22
+TACGAGTATG
+>MID23
+TACTCTCGTG
+>MID24
+TAGAGACGAG
+>MID25
+TCGTCGCTCG
+>MID26
+ACATACGCGT
+>MID27
+ACGCGAGTAT
+>MID28
+ACTACTATGT
+>MID68
+TCGCTGCGTA
+>MID30
+AGACTATACT
+>MID31
+AGCGTCGTCT
+>MID32
+AGTACGCTAT
+>MID33
+ATAGAGTACT
+>MID34
+CACGCTACGT
+>MID35
+CAGTAGACGT
+>MID36
+CGACGTGACT
+>MID37
+TACACACACT
+>MID38
+TACACGTGAT
+>MID39
+TACAGATCGT
+>MID40
+TACGCTGTCT
+>MID69
+TCTGACGTCA
+>MID42
+TCGATCACGT
+>MID43
+TCGCACTAGT
+>MID44
+TCTAGCGACT
+>MID45
+TCTATACTAT
+>MID46
+TGACGTATGT
+>MID47
+TGTGAGTAGT
+>MID48
+ACAGTATATA
+>MID49
+ACGCGATCGA
+>MID50
+ACTAGCAGTA
+>MID67
+TCGATAGTGA

data/skeleton/remove_mids/lib/find_mids.rb

@@ -0,0 +1,191 @@
+require 'scbi_blast'
+require 'global_match'
+# require 'json'
+
+# Module to find Mids in a set of sequences
+module FindMids
+
+  # find mids using blast+ as an external tool
+  def find_mid_with_blast(seqs)
+    t=Time.now
+
+    # Create blast machine agains mid database
+    blast = BatchBlast.new("-db #{File.expand_path(File.join(File.dirname(__FILE__),'db/mids.fasta'))}",'blastn'," -task blastn-short    -perc_identity 95 -max_target_seqs 4 ")  #get mids
+
+    # build fastas to blast
+    fastas=[]
+
+    seqs.each do |name,fasta,qual,comments|
+      fastas.push ">"+name
+      fastas.push fasta
+    end
+
+    # execute blast
+    blast_table_results = blast.do_blast(fastas)
+
+    # puts blast_table_results.inspect
+
+    # Iterate over blast results and sequences
+    i=0
+    seqs.each do |name,fasta,qual,comments|
+      parse_seq(blast_table_results.querys[i],name,fasta,qual,comments)
+      i+=1
+    end
+
+    elapsed=Time.now-t
+
+    puts "T:#{elapsed}, rate#{elapsed/seqs.count}"
+
+  end
+
+
+  # parse blast results and sequences to remove found MIDS
+  def parse_seq(query,name,fasta,qual,comments)
+
+    # find_polys('TN',fasta)
+    #     find_polys('AN',fasta)
+
+    query.hits.each do |found_mid|
+
+      if found_mid.align_len>1
+
+        # modify comments by appending removed mid
+        comments << found_mid.subject_id
+
+        # keep fasta from pos to end
+        fasta.slice!(0, found_mid.q_beg + found_mid.align_len)
+
+        # keep qual from pos to end
+        qual.slice!(0, found_mid.q_beg + found_mid.align_len)
+        break
+      end
+    end
+  end
+
+  def find_mid_without_blast(seqs)
+    # those are the mids found in database
+    t=Time.now
+
+    mids={}
+    mids['RL1']='ACACGACGACT'
+    mids['RL2']='ACACGTAGTAT'
+    mids['RL3']='ACACTACTCGT'
+    mids['RL4']='ACGACACGTAT'
+    mids['RL5']='ACGAGTAGACT'
+    mids['RL6']='ACGCGTCTAGT'
+    mids['RL7']='ACGTACACACT'
+    mids['RL8']='ACGTACTGTGT'
+    mids['RL9']='ACGTAGATCGT'
+    mids['RL10']='ACTACGTCTCT'
+    mids['RL11']='ACTATACGAGT'
+    mids['RL12']='ACTCGCGTCGT'
+    mids['MID1']='ACGAGTGCGT'
+    mids['MID2']='ACGCTCGACA'
+    mids['MID3']='AGACGCACTC'
+    mids['MID5']='ATCAGACACG'
+    mids['MID6']='ATATCGCGAG'
+    mids['MID7']='CGTGTCTCTA'
+    mids['MID8']='CTCGCGTGTC'
+    mids['MID10']='TCTCTATGCG'
+    mids['MID11']='TGATACGTCT'
+    mids['MID13']='CATAGTAGTG'
+    mids['MID14']='CGAGAGATAC'
+    mids['MID15']='ATACGACGTA'
+    mids['MID16']='TCACGTACTA'
+    mids['MID17']='CGTCTAGTAC'
+    mids['MID18']='TCTACGTAGC'
+    mids['MID19']='TGTACTACTC'
+    mids['MID20']='ACGACTACAG'
+    mids['MID21']='CGTAGACTAG'
+    mids['MID22']='TACGAGTATG'
+    mids['MID23']='TACTCTCGTG'
+    mids['MID24']='TAGAGACGAG'
+    mids['MID25']='TCGTCGCTCG'
+    mids['MID26']='ACATACGCGT'
+    mids['MID27']='ACGCGAGTAT'
+    mids['MID28']='ACTACTATGT'
+    mids['MID68']='TCGCTGCGTA'
+    mids['MID30']='AGACTATACT'
+    mids['MID31']='AGCGTCGTCT'
+    mids['MID32']='AGTACGCTAT'
+    mids['MID33']='ATAGAGTACT'
+    mids['MID34']='CACGCTACGT'
+    mids['MID35']='CAGTAGACGT'
+    mids['MID36']='CGACGTGACT'
+    mids['MID37']='TACACACACT'
+    mids['MID38']='TACACGTGAT'
+    mids['MID39']='TACAGATCGT'
+    mids['MID40']='TACGCTGTCT'
+    mids['MID69']='TCTGACGTCA'
+    mids['MID42']='TCGATCACGT'
+    mids['MID43']='TCGCACTAGT'
+    mids['MID44']='TCTAGCGACT'
+    mids['MID45']='TCTATACTAT'
+    mids['MID46']='TGACGTATGT'
+    mids['MID47']='TGTGAGTAGT'
+    mids['MID48']='ACAGTATATA'
+    mids['MID49']='ACGCGATCGA'
+    mids['MID50']='ACTAGCAGTA'
+    mids['MID67']='TCGATAGTGA'
+
+    # for each sequence
+    seqs.each do |name,fasta,qual,comment|
+      
+      # find all mids
+      mids.each do |mid_name,mid|
+        # puts "."
+        # find a known MID position
+        found_mid=fasta[0..20].lcs(mid)
+        # puts "."
+        # puts pos.to_json
+        if found_mid.length>5
+
+          pos=fasta[0..20].index(found_mid)
+          # puts found_mid,pos
+          # keep fasta from pos to end
+          fasta.slice!(0,pos+found_mid.length)
+
+          # keep qual from pos to end
+          qual.slice!(0,pos+found_mid.length)
+
+          comment << "mid_name #{mid_name}\n"
+          # puts comment
+          break
+        end
+      end
+    end
+
+    elapsed=Time.now-t
+
+    puts "T:#{elapsed}, rate#{elapsed/seqs.count}"
+
+  end
+
+
+  def do_dummy_calculation
+    numer_of_calcs=250000
+
+    t=Time.now
+
+    x1=1
+    x2=1
+
+    # do a loop with calculations
+    numer_of_calcs.times do |i|
+      x=x1+x2
+
+      x1=x2
+      x2=x
+
+      # puts some info at regular intervals
+      if (i % 100000)==0
+        puts "Calculated #{i} by thread #{n}"
+      end
+    end
+    puts Time.now-t
+
+  end
+
+
+
+end