scbi_mapreduce 0.0.40 → 0.0.45

data/skeleton/old/dummy_calculations/main.rb

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+
+$: << File.join(File.dirname(__FILE__))
+
+# load required libraries
+require 'scbi_mapreduce'
+require 'my_worker_manager'
+
+# listen on all ips at port 50000
+ip='0.0.0.0'
+port = 50000
+
+# set number of workers. You can also provide an array with worker names.
+# Those workers names can be read from a file produced by the existing
+# queue system, if any.
+
+
+
+workers = 4
+
+# read optional workers parameter
+input_workers = ARGV.shift
+if !input_workers.nil?
+  # if it is a file
+  if File.exists?(input_workers)
+    # read workers into array
+    workers=File.read(input_workers).split("\n").map{|w| w.chomp}
+  else
+    # workers is a number
+    workers = input_workers.to_i
+  end
+end
+
+# we need the path to my_worker in order to launch it when necessary
+custom_worker_file = File.join(File.dirname(__FILE__),'my_worker.rb')
+
+# initialize the work manager. Here you can pass parameters like file names
+MyWorkerManager.init_work_manager
+
+# launch processor server
+mgr = ScbiMapreduce::Manager.new(ip, port, workers, MyWorkerManager, custom_worker_file, STDOUT)
+
+# you can set additional properties
+# =================================
+
+# if you want basic checkpointing. Some performance drop should be expected
+# mgr.checkpointing=true
+
+# if you want to keep the order of input data. Some performance drop should be expected
+# mgr.keep_order=true
+
+# you can set the size of packets of data sent to workers
+mgr.chunk_size=1
+
+# start processing
+mgr.start_server
+
+# this line is reached when all data has been processed
+puts "Program finished"

data/skeleton/old/dummy_calculations/my_worker.rb

@@ -0,0 +1,56 @@
+$: << File.join(File.dirname(__FILE__),'lib')
+
+require 'calculations'
+include Calculations
+
+# MyWorker defines the behaviour of workers.
+# Here is where the real processing takes place
+class MyWorker < ScbiMapreduce::Worker
+
+  # starting_worker method is called one time at initialization
+  # and allows you to initialize your variables
+  def starting_worker
+
+    # You can use worker logs at any time in this way:
+    # $WORKER_LOG.info "Starting a worker"
+
+  end
+
+
+  # receive_initial_config is called only once just after
+  # the first connection, when initial parameters are
+  # received from manager
+  def receive_initial_config(parameters)
+
+    # Reads the parameters
+
+    # You can use worker logs at any time in this way:
+    # $WORKER_LOG.info "Params received"
+
+    # save received parameters, if any
+    # @params = parameters
+  end
+
+
+  # process_object method is called for each received object.
+  # Be aware that objs is always an array, and you must iterate
+  # over it if you need to process it independently
+  #
+  # The value returned here will be received by the work_received
+  # method at your worker_manager subclass.
+  def process_object(objs)
+
+    # iterate over all objects received
+    # objs.each do |obj|
+      # convert to uppercase
+    do_dummy_calculations
+    # end
+    
+    # return objs back to manager
+    return objs
+  end
+
+  def closing_worker
+
+  end
+end

data/skeleton/old/dummy_calculations/my_worker_manager.rb

@@ -0,0 +1,52 @@
+require 'json'
+
+# MyWorkerManager class is used to implement the methods
+# to send and receive the data to or from workers
+class MyWorkerManager < ScbiMapreduce::WorkManager
+
+  # init_work_manager is executed at the start, prior to any processing.
+  # You can use init_work_manager to initialize global variables, open files, etc...
+  # Note that an instance of MyWorkerManager will be created for each
+  # worker connection, and thus, all global variables here should be
+  # class variables (starting with @@)
+  def self.init_work_manager
+    
+    # execute dummy_calc in workers @remaining_data times
+    @@remaining_data = 1000
+  end
+
+  # end_work_manager is executed at the end, when all the process is done.
+  # You can use it to close files opened in init_work_manager
+  def self.end_work_manager
+
+  end
+
+  # worker_initial_config is used to send initial parameters to workers.
+  # The method is executed once per each worker
+  def worker_initial_config
+
+  end
+
+  # next_work method is called every time a worker needs a new work
+  # Here you can read data from disk
+  # This method must return the work data or nil if no more data is available
+  def next_work
+    @@remaining_data -= 1
+
+    e = @@remaining_data
+
+    e = nil if @@remaining_data<=0
+    
+    return e
+
+  end
+
+
+  # work_received is executed each time a worker has finished a job.
+  # Here you can write results down to disk, perform some aggregated statistics, etc...
+  def work_received(results)
+
+    # write_data_to_disk(results)
+  end
+
+end

data/skeleton/old/dummy_calculations/threads_implementation.rb

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+# load required libraries
+
+# modify include path
+$: << File.join(File.dirname(__FILE__),'lib')
+
+require 'thread_pool'
+require 'calculations'
+include Calculations
+
+
+@pool=ThreadPool.new(4)
+
+# process
+  
+times_to_calculate=1000
+
+times_to_calculate.times do
+
+  @pool.process {do_dummy_calculations}
+
+end
+
+
+puts "wait"
+@pool.join
+puts "final"
+

data/skeleton/old/sequences_blast/README.txt

@@ -0,0 +1,31 @@
+A simple scbi_mapreduce application demo
+========================================
+
+This application is a basic sequence processing template. It processes all 
+sequences in fastq_file (a file in FastQ format) removing a MIDs from it. It 
+needs some external requisites to work:
+
+EXTERNAL REQUISITES
+===================
+
+* Blast+ 2.2.24 or greater installed
+* scbi_blast gem installed
+
+
+At lib/db you can find a preformated MID database for blast+ (formatted with makeblastdb).
+
+You can modify the files to perform more complicated processing. 
+There are other templates available, you can list them by issuing this command:
+
+scbi_mapreduce
+
+You can launch the application right now with the following command:
+
+  ruby main.rb fastq_file
+
+A server and some workers will be launched, and all sequences in fastq_file will
+be processed in blocks of 100 sequences.
+
+A sequential example is also provided, you can launch it by issuing:
+
+ruby linear_implementation.rb fastq_file

data/{test/drb_test/main.rb → skeleton/old/sequences_blast/launch_only_workers.rb}

@@ -4,7 +4,7 @@ $: << File.dirname(__FILE__)
 
 require "logger"
 
-$: << File.expand_path('../../lib')
+$: << '/Users/dariogf/progs/ruby/gems/scbi_mapreduce/lib'
 
 require 'scbi_mapreduce'
 require 'my_worker_manager'
@@ -13,21 +13,17 @@ require 'my_worker_manager'
 $LOG = Logger.new(STDOUT)
 $LOG.datetime_format = "%Y-%m-%d %H:%M:%S"
 
-ip='0.0.0.0'
+ip='10.247.255.5'
 port = 50000
 workers = 8
 
-workers = File.expand_path('workers')
-
 custom_worker_file = File.join(File.dirname(__FILE__),'my_worker.rb')
 
 $LOG.info 'Starting server'
 
-MyWorkerManager.init_work_manager
+
+worker_launcher = ScbiMapreduce::WorkerLauncher.new(ip,port, workers, custom_worker_file, STDOUT)
+worker_launcher.launch_workers_and_wait
 
 # launch processor server
-mgr = ScbiMapreduce::Manager.new(ip,port, workers, MyWorkerManager,custom_worker_file, STDOUT)
-# mgr.checkpointing=false
-# mgr.keep_order=true
-mgr.start_server
-$LOG.info 'Closing server'
+$LOG.info 'Closing workers'

data/skeleton/old/sequences_blast/lib/db/mids.fasta

@@ -0,0 +1,64 @@
+>RL1
+ACACGACGACT
+>RL2
+ACACGTAGTAT
+>RL3
+ACACTACTCGT
+>RL4
+ACGACACGTAT
+>RL5
+ACGAGTAGACT
+>RL6
+ACGCGTCTAGT
+>RL7
+ACGTACACACT
+>RL8
+ACGTACTGTGT
+>RL9
+ACGTAGATCGT
+>RL10
+ACTACGTCTCT
+>RL11
+ACTATACGAGT
+>RL12
+ACTCGCGTCGT
+
+
+>MID1
+ACGAGTGCGT
+
+>MID2
+ACGCTCGACA
+
+>MID3
+AGACGCACTC
+
+>MID4
+AGCACTGTAG
+
+>MID5
+ATCAGACACG
+
+>MID6
+ATATCGCGAG
+
+>MID7
+CGTGTCTCTA
+
+>MID8
+CTCGCGTGTC
+
+>MID9
+TAGTATCAGC
+
+>MID10
+TCTCTATGCG
+
+>MID11
+TGATACGTCT
+
+>MID12
+TACTGAGCTA
+
+
+

data/skeleton/old/sequences_blast/lib/db/mids.fasta.nsd

@@ -0,0 +1,48 @@
+lcl|mid112
+lcl|mid1021
+lcl|mid1122
+lcl|mid1223
+lcl|mid213
+lcl|mid314
+lcl|mid415
+lcl|mid516
+lcl|mid617
+lcl|mid718
+lcl|mid819
+lcl|mid920
+lcl|rl10
+lcl|rl109
+lcl|rl1110
+lcl|rl1211
+lcl|rl21
+lcl|rl32
+lcl|rl43
+lcl|rl54
+lcl|rl65
+lcl|rl76
+lcl|rl87
+lcl|rl98
+mid112
+mid1021
+mid1122
+mid1223
+mid213
+mid314
+mid415
+mid516
+mid617
+mid718
+mid819
+mid920
+rl10
+rl109
+rl1110
+rl1211
+rl21
+rl32
+rl43
+rl54
+rl65
+rl76
+rl87
+rl98

data/skeleton/old/sequences_blast/lib/find_mids.rb

@@ -0,0 +1,134 @@
+require 'scbi_blast'
+# require 'json'
+
+# Module to find Mids in a set of sequences
+module FindMids
+
+  # find mids using blast+ as an external tool
+  def find_mid_with_blast(seqs)
+    t=Time.now
+
+    # Create blast machine agains mid database
+    blast = BatchBlast.new("-db #{File.expand_path(File.join(File.dirname(__FILE__),'db/mids.fasta'))}",'blastn'," -task blastn-short    -perc_identity 95 -max_target_seqs 4 ")  #get mids
+
+    # build fastas to blast
+    fastas=[]
+
+    seqs.each do |name,fasta,qual,comments|
+      fastas.push ">"+name
+      fastas.push fasta
+    end
+
+    # execute blast
+    blast_table_results = blast.do_blast(fastas)
+
+    puts blast_table_results.inspect
+
+    # Iterate over blast results and sequences
+    i=0
+    seqs.each do |name,fasta,qual,comments|
+      parse_seq(blast_table_results.querys[i],name,fasta,qual,comments)
+      i+=1
+    end
+
+    puts Time.now-t
+
+  end
+
+
+  # parse blast results and sequences to remove found MIDS
+  def parse_seq(query,name,fasta,qual,comments)
+
+    query.hits.each do |found_mid|
+
+      if found_mid.align_len>7
+
+        # modify comments by appending removed mid
+        comments << found_mid.subject_id
+
+        # keep fasta from pos to end
+        fasta.slice!(0, found_mid.q_beg + found_mid.align_len)
+
+        # keep qual from pos to end
+        qual.slice!(0, found_mid.q_beg + found_mid.align_len)
+        break
+      end
+    end
+  end
+
+  def find_mid_without_blast(seqs)
+    # those are the mids found in database
+
+    mids={}
+    mids['RL1']='ACACGACGACT'
+    mids['RL2']='ACACGTAGTAT'
+    mids['RL3']='ACACTACTCGT'
+    mids['RL4']='ACGACACGTAT'
+    mids['RL5']='ACGAGTAGACT'
+    mids['RL6']='ACGCGTCTAGT'
+    mids['RL7']='ACGTACACACT'
+    mids['RL8']='ACGTACTGTGT'
+    mids['RL9']='ACGTAGATCGT'
+    mids['RL10']='ACTACGTCTCT'
+    mids['RL11']='ACTATACGAGT'
+    mids['RL12']='ACTCGCGTCGT'
+    mids['MID1']='ACGAGTGCGT'
+    mids['MID2']='ACGCTCGACA'
+    mids['MID3']='AGACGCACTC'
+    mids['MID4']='AGCACTGTAG'
+    mids['MID5']='ATCAGACACG'
+    mids['MID6']='ATATCGCGAG'
+    mids['MID7']='CGTGTCTCTA'
+    mids['MID8']='CTCGCGTGTC'
+    mids['MID9']='TAGTATCAGC'
+    mids['MID10']='TCTCTATGCG'
+    mids['MID11']='TGATACGTCT'
+    mids['MID12']='TACTGAGCTA'
+
+    mids.each do |mid_name,mid|
+
+      seqs.each do |name,fasta,qual,comment|
+
+        # find a known MID position
+        pos=fasta.upcase.index(mid)
+
+        if pos
+
+          # keep fasta from pos to end
+          fasta.slice!(0,pos+mid.length)
+
+          # keep qual from pos to end
+          qual.slice!(0,pos+mid.length)
+
+        end
+      end
+    end
+
+  end
+  
+  
+  def do_dummy_calculation
+    numer_of_calcs=250000
+
+    t=Time.now
+
+    x1=1
+    x2=1
+
+    # do a loop with calculations
+    numer_of_calcs.times do |i|
+      x=x1+x2
+
+      x1=x2
+      x2=x
+
+      # puts some info at regular intervals
+      if (i % 100000)==0
+        puts "Calculated #{i} by thread #{n}"
+      end
+    end
+    puts Time.now-t
+
+  end
+
+end