scbi_mapreduce 0.0.29 → 0.0.31

data/History.txt

@@ -1,3 +1,11 @@
+=== 0.0.31 2011-06-17
+
+Total connected workers stats
+
+=== 0.0.30 2011-06-17
+
+Report local workers
+
 === 0.0.29 2011-06-13
 
 First rubygems release

data/lib/scbi_mapreduce.rb

@@ -4,7 +4,7 @@ $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname
 # $: << File.join(File.dirname(__FILE__),File.basename(__FILE__,File.extname(__FILE__)))
 
 module ScbiMapreduce
-   VERSION = '0.0.29'
+   VERSION = '0.0.31'
 end
 
 require 'scbi_mapreduce/manager'

data/lib/scbi_mapreduce/manager.rb

@@ -68,6 +68,9 @@ module ScbiMapreduce
       FileUtils.mkdir_p(File.dirname(log_file)) if ((log_file!=STDOUT) && (!File.exists?(File.dirname(log_file))))
       $SERVER_LOG = Logger.new(log_file)
 
+      $SERVER_LOG.info("Local workers: #{@workers}")
+      $SERVER_LOG.info("Remote workers: #{@worker_names}")
+
 
       $SERVER_LOG.datetime_format = "%Y-%m-%d %H:%M:%S"
 

data/lib/scbi_mapreduce/work_manager.rb

@@ -106,6 +106,7 @@ module ScbiMapreduce
       @@count = 0
       @@chunk_count = 0
       @@workers = 0
+      @@max_workers = 0
       @@error_count = 0
       @@running_jobs=[]
       # @@compress=true
@@ -254,7 +255,7 @@ module ScbiMapreduce
 
     def post_init
       @@workers += 1
-
+      @@max_workers +=1
       # when first worker is connected, do special config
       if @@workers == 1
         @@total_seconds = Time.now
@@ -396,6 +397,8 @@ module ScbiMapreduce
 
         $SERVER_LOG.info  "Number of errors: #{@@error_count}"
         $SERVER_LOG.info  "Chunk size: #{@@chunk_size}"
+        $SERVER_LOG.info  "Total connected workers: #{@@max_workers}"
+        
 
 
       end

data/lib/scbi_mapreduce/worker_launcher.rb

@@ -33,6 +33,8 @@ module ScbiMapreduce
     end
 
     def launch_workers
+      # TODO - si aqui falla algo, no peta, se bloquea
+      $LAUNCHER_LOG.info "Launching #{@workers} local workers"
       if @workers > 0
         $LAUNCHER_LOG.info "Connecting #{@workers} local workers to #{@server_ip}:#{@server_port}"
         threads = []

data/skeleton/remove_mids/main.rb

@@ -5,14 +5,11 @@ $: << File.dirname(__FILE__)
 # load required libraries
 require 'scbi_mapreduce'
 
-# in order to load fastq files
-require 'scbi_fastq'
-
+require 'scbi_fastq' #to load fastq files
 require 'my_worker_manager.rb'
 
 # check arguments
 if ARGV.count != 3
-  
   puts "Usage #{File.basename($0)} fastq_file workers chunk_size"
   puts ""
   puts "#{File.basename($0)} iterates over all sequences in fastq_file (a file in FastQ format) and removes a MID (barcode) from it"  
@@ -26,31 +23,19 @@ if !File.exists?(fastq_file_path)
   exit
 end 
 
-# listen on all ips at port 50000
+# listen on ip starting with 10.243 at first available port
 server_ip='10.243'
-ip_list = Socket.ip_address_list.select{|e| e.ipv4?}.map{|e| e.ip_address}
-
-ip=ip_list.select{|ip| ip.index(server_ip)==0}.first
-
-if !ip
-  ip='0.0.0.0'
-end
-
-  port = 0
+port = 0
 
 # set number of workers. You can also provide an array with worker names.
 # Those workers names can be read from a file produced by the existing
 # queue system, if any.
-
-
-
 workers = 4
 
 # read optional workers parameter
-  input_workers = ARGV.shift
+input_workers = ARGV.shift
 if !input_workers.nil?
-  # if it is a file
-  if File.exists?(input_workers)
+  if File.exists?(input_workers) # if it is a file
     # read workers into array
     workers=File.read(input_workers).split("\n").map{|w| w.chomp}
   else
@@ -59,6 +44,7 @@ if !input_workers.nil?
   end
 end
 
+# read chunk size from args
 chunk_size = ARGV.shift.to_i
 
 # we need the path to my_worker in order to launch it when necessary
@@ -68,7 +54,7 @@ custom_worker_file = File.join(File.dirname(__FILE__),'my_worker.rb')
 MyWorkerManager.init_work_manager(fastq_file_path)
 
 # launch processor server
-mgr = ScbiMapreduce::Manager.new(ip, port, workers, MyWorkerManager, custom_worker_file, STDOUT,'~seqtrimnext/init_env')
+mgr = ScbiMapreduce::Manager.new(server_ip, port, workers, MyWorkerManager, custom_worker_file, STDOUT,'~seqtrimnext/init_env')
 
 # you can set additional properties
 # =================================

metadata

@@ -2,7 +2,7 @@
 name: scbi_mapreduce
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.29
+  version: 0.0.31
 platform: ruby
 authors: 
 - Dario Guerrero
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-13 00:00:00 Z
+date: 2011-06-17 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: eventmachine