scbi_mapreduce 0.0.29

data/skeleton/simple/my_worker.rb

@@ -0,0 +1,53 @@
+# MyWorker defines the behaviour of workers.
+# Here is where the real processing takes place
+class MyWorker < ScbiMapreduce::Worker
+
+  # starting_worker method is called one time at initialization
+  # and allows you to initialize your variables
+  def starting_worker
+
+    # You can use worker logs at any time in this way:
+    # $WORKER_LOG.info "Starting a worker"
+
+  end
+
+
+  # receive_initial_config is called only once just after
+  # the first connection, when initial parameters are
+  # received from manager
+  def receive_initial_config(parameters)
+
+    # Reads the parameters
+
+    # You can use worker logs at any time in this way:
+    # $WORKER_LOG.info "Params received"
+
+    # save received parameters, if any
+    # @params = parameters
+  end
+
+
+  # process_object method is called for each received object.
+  # Be aware that objs is always an array, and you must iterate
+  # over it if you need to process it independently
+  #
+  # The value returned here will be received by the work_received
+  # method at your worker_manager subclass.
+  def process_object(objs)
+
+    # iterate over all objects received
+    objs.each do |obj|
+
+      # convert to uppercase
+      obj.upcase!
+    end
+
+    # return objs back to manager
+    return objs
+  end
+
+  # called once, when the worker is about to be closed
+  def closing_worker
+
+  end
+end

data/skeleton/simple/my_worker_manager.rb

@@ -0,0 +1,55 @@
+require 'json'
+
+# MyWorkerManager class is used to implement the methods
+# to send and receive the data to or from workers
+class MyWorkerManager < ScbiMapreduce::WorkManager
+
+  # init_work_manager is executed at the start, prior to any processing.
+  # You can use init_work_manager to initialize global variables, open files, etc...
+  # Note that an instance of MyWorkerManager will be created for each
+  # worker connection, and thus, all global variables here should be
+  # class variables (starting with @@)
+  def self.init_work_manager
+    
+    # use 200000 strings
+    @@remaining_data = 200000
+    
+    # of 1024 characters each
+    @@basic_string='a'*1024
+
+  end
+
+  # end_work_manager is executed at the end, when all the process is done.
+  # You can use it to close files opened in init_work_manager
+  def self.end_work_manager
+
+  end
+
+  # worker_initial_config is used to send initial parameters to workers.
+  # The method is executed once per each worker
+  def worker_initial_config
+
+  end
+
+  # next_work method is called every time a worker needs a new work
+  # Here you can read data from disk
+  # This method must return the work data or nil if no more data is available
+  def next_work
+    @@remaining_data -= 1
+
+    e = @@basic_string
+
+    e = nil if @@remaining_data<0
+    return e
+
+  end
+
+
+  # work_received is executed each time a worker has finished a job.
+  # Here you can write results down to disk, perform some aggregated statistics, etc...
+  def work_received(results)
+
+    # write_data_to_disk(results)
+  end
+
+end

data/test/drb_test/main.rb

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+$: << File.dirname(__FILE__)
+
+require "logger"
+
+# $: << '/Users/dariogf/progs/ruby/gems/scbi_mapreduce/lib'
+
+require 'scbi_mapreduce'
+require 'my_worker_manager'
+
+
+$LOG = Logger.new(STDOUT)
+$LOG.datetime_format = "%Y-%m-%d %H:%M:%S"
+
+ip='0.0.0.0'
+port = 50000
+workers = 8
+
+custom_worker_file = File.join(File.dirname(__FILE__),'my_worker.rb')
+
+$LOG.info 'Starting server'
+
+MyWorkerManager.init_work_manager
+
+# launch processor server
+mgr = ScbiMapreduce::Manager.new(ip,port, workers, MyWorkerManager,custom_worker_file, STDOUT)
+# mgr.checkpointing=false
+# mgr.keep_order=true
+mgr.start_server
+$LOG.info 'Closing server'

data/test/drb_test/my_worker.rb

@@ -0,0 +1,36 @@
+# require 'scbi_mapreduce'
+require 'json'
+
+class MyWorker < ScbiMapreduce::Worker
+
+	def starting_worker
+
+			# $WORKER_LOG.info "Loading actions"
+	rescue Exception => e
+		puts (e.message+ e.backtrace.join("\n"))
+	
+	end
+
+		
+    def receive_initial_config(obj)
+
+				# Reads the parameters
+				# $WORKER_LOG.info "Params received: #{obj.to_json}"				
+				@params = obj
+				
+    
+    end
+
+		
+		def process_object(obj)
+         #return `echo #{obj} | tr aeiou AEIOU`.chomp
+					# sleep 1
+         return obj
+		end
+    
+		
+		def closing_worker
+			 
+		end		
+end
+

data/test/drb_test/my_worker_manager.rb

@@ -0,0 +1,41 @@
+require 'json'
+
+class MyWorkerManager < ScbiMapreduce::WorkManager
+
+		def self.init_work_manager
+  			 @@params = {:algo=> 5}
+				 @@datos = 200000
+				 # @@fi=File.open(/tmp
+		end
+		
+		def self.end_work_manager
+
+		end
+		
+		def worker_initial_config
+				return @@params
+		end
+		 
+		# this method is called every time a worker needs a new work
+		# Return the work data or nil if no more data is available
+  	def next_work
+	        @@datos -= 1
+			if (@@datos%10000)==0
+			  puts "=> #{@@datos}"
+		  end
+		  
+			e = @@datos
+			e = nil if @@datos<0
+			return e
+			
+  	end
+  	
+  	def work_received(obj)
+  	
+  		# puts obj
+      # write_seq_to_disk(seq)
+  	    
+  	end 
+  	
+end
+

data/test/drb_test/scbi_drb_checkpoint

@@ -0,0 +1 @@
+95598

data/test/drb_test/scbi_mapreduce_checkpoint

@@ -0,0 +1 @@
+199909

data/test/test_helper.rb

@@ -0,0 +1,3 @@
+require 'stringio'
+require 'test/unit'
+require File.dirname(__FILE__) + '/../lib/scbi_mapreduce'

data/test/test_scbi_drb.rb

@@ -0,0 +1,11 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class TestScbiDrb < Test::Unit::TestCase
+
+  def setup
+  end
+  
+  def test_truth
+    assert true
+  end
+end

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification 
+name: scbi_mapreduce
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.29
+platform: ruby
+authors: 
+- Dario Guerrero
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-06-13 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.12.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.8.0
+  type: :development
+  version_requirements: *id002
+description: scbi_mapreduce brings parallel and distributed computing capabilities to your code, with a very easy to use framework that allows you to exploit your clustered or cloud computational resources.
+email: 
+- dariogf@gmail.com
+executables: 
+- scbi_mapreduce
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- PostInstall.txt
+- skeleton/simple/README.txt
+- skeleton/remove_mids/README.txt
+- skeleton/dummy_calcs/README.txt
+files: 
+- History.txt
+- lib/scbi_mapreduce/error_handler.rb
+- lib/scbi_mapreduce/main_worker.rb
+- lib/scbi_mapreduce/manager.rb
+- lib/scbi_mapreduce/work_manager.rb
+- lib/scbi_mapreduce/worker.rb
+- lib/scbi_mapreduce/zlib_serializer.rb
+- lib/scbi_mapreduce/worker_launcher.rb
+- lib/scbi_mapreduce.rb
+- Manifest.txt
+- PostInstall.txt
+- Rakefile
+- README.rdoc
+- script/console
+- script/destroy
+- script/generate
+- test/drb_test/main.rb
+- test/drb_test/my_worker.rb
+- test/drb_test/my_worker_manager.rb
+- test/drb_test/scbi_drb_checkpoint
+- test/drb_test/scbi_mapreduce_checkpoint
+- test/test_helper.rb
+- test/test_scbi_drb.rb
+- bin/scbi_mapreduce
+- skeleton/simple/main.rb
+- skeleton/simple/my_worker.rb
+- skeleton/simple/my_worker_manager.rb
+- skeleton/simple/README.txt
+- skeleton/remove_mids/launch_only_workers.rb
+- skeleton/remove_mids/lib/db/mids.fasta
+- skeleton/remove_mids/lib/find_mids.rb
+- skeleton/remove_mids/lib/global_match.rb
+- skeleton/remove_mids/linear_implementation.rb
+- skeleton/remove_mids/main.rb
+- skeleton/remove_mids/my_worker.rb
+- skeleton/remove_mids/my_worker_manager.rb
+- skeleton/remove_mids/README.txt
+- skeleton/dummy_calcs/lib/calculations.rb
+- skeleton/dummy_calcs/lib/thread_pool.rb
+- skeleton/dummy_calcs/linear_implementation.rb
+- skeleton/dummy_calcs/main.rb
+- skeleton/dummy_calcs/my_worker.rb
+- skeleton/dummy_calcs/my_worker_manager.rb
+- skeleton/dummy_calcs/README.txt
+- skeleton/dummy_calcs/threads_implementation.rb
+homepage: http://www.scbi.uma.es/downloads
+licenses: []
+
+post_install_message: PostInstall.txt
+rdoc_options: 
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: scbi_mapreduce
+rubygems_version: 1.7.2
+signing_key: 
+specification_version: 3
+summary: scbi_mapreduce brings parallel and distributed computing capabilities to your code, with a very easy to use framework that allows you to exploit your clustered or cloud computational resources.
+test_files: 
+- test/test_helper.rb
+- test/test_scbi_drb.rb