madrox-cluster 0.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+--- 
+SHA1: 
+  data.tar.gz: 5c3dad986634c21d65fd78e0ea57689f5362e0a8
+  metadata.gz: 6bec33f51f16851df54d14d8a8fe56591ce83806
+SHA512: 
+  data.tar.gz: d8e0028ab40eec52ac32ad4f0d74d19328d7f2c1e38efe1b2ca7315d27c095e4cfd2316f3603683d024c734d5950549ef517d14c4b1598a2551e30b159d8d511
+  metadata.gz: 397ccd35cb71082be4d7e5f4e2809dd48826ca4e5c158d7d1d7609d61a12df829ec40ed1c9e0cf9b6d3770a2422c17b4b309dbbeaa9455bc1eeedabc971382d4

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in madrox-cluster.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/Readme.md

@@ -0,0 +1,68 @@
+Madrox [![Code Climate](https://codeclimate.com/github/iurimatias/madrox-cluster.png)](https://codeclimate.com/github/iurimatias/madrox-cluster)
+======
+
+Easily distribute any code across multiple servers
+
+Install
+=======
+
+```Bash
+gem install madrox-cluster
+```
+
+Usage
+=====
+
+###Starting a Server
+
+```Bash
+madrox 127.0.0.1 5000
+```
+
+###Adding servers
+
+```Ruby
+#configure a server
+Madrox.config(["server_1:5000", "server_1:5001", "server_2:5000", "server_2:5001"])
+```
+
+###Executing a block
+
+
+```Ruby
+#Distribute processing across the servers
+result = Madrox.collect([35, 30, 35, 37, 25, 30]) do |x|
+  def fib(n)
+    n<=1 ? n : fib(n-2) + fib(n-1)
+  end
+  fib(x)
+end
+```
+
+###Register code in advance
+
+```Ruby
+CalcClass = Proc.new { |x|
+  class Calc
+    def fib(n)
+      n<=1 ? n : fib(n-2) + fib(n-1)
+    end
+  end
+
+  Calc
+}
+
+#stores this class in all servers
+Madrox.register("Calc", CalcClass)
+
+result = Madrox.collect([35, 30, 35, 37, 25, 30]) do |x|
+  Calc.new.fib(x)
+end
+```
+
+Disclaimer
+=====
+
+This gem is provided as is - therefore, the creators and contributors of this gem are not responsible for any damages that may result from its usage. Although Authentication and SSL may be added later, Madrox is experimental and is meant to be used in private trusted local networks. Use at your own risk.
+
+Madrox is an experiment. For a more solid solution please check [dRuby](http://www.ruby-doc.org/stdlib-1.9.3/libdoc/drb/rdoc/DRb.html)

data/bin/madrox

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+require 'madrox-cluster/server/server'
+
+hostname = ARGV[0] || "0.0.0.0"
+port     = ARGV[1] || 2000
+
+Madrox::Server.start(hostname, port)
+

data/lib/madrox-cluster/host.rb

@@ -0,0 +1,38 @@
+module Madrox
+  class Host
+    attr_accessor :jobs, :connection
+
+    def initialize(hostname, port)
+      @hostname = hostname
+      @port     = port
+      @jobs     = 0
+      @semaphore = Mutex.new
+    end
+
+    def connect
+      @connection = TCPSocket.new(@hostname, @port)
+    rescue Errno::ECONNREFUSED => e
+      raise "problem connecting to #{@hostname}:#{@port}"
+    end
+
+    def send(package, should_get_reply=true)
+      @semaphore.lock
+      connect
+      @jobs += 1
+      @connection.sendmsg package #add \n
+
+      result = should_get_reply ? @connection.gets.chop : nil
+      @jobs -= 1
+      @semaphore.unlock
+
+      result
+    #rescue
+    #  retry
+    end
+
+    def close_connection
+      @connection.close
+    end
+
+  end
+end

data/lib/madrox-cluster/hosts_manager.rb

@@ -0,0 +1,33 @@
+module Madrox
+  class HostsManager
+
+    def self.add_hosts(host_list)
+      host_list.each do |host|
+        hostname, port = host.split(":")
+        add_host(hostname, port)
+      end
+    end
+
+    def self.add_host(hostname, port)
+      @@hosts ||= []
+      @@hosts << Host.new(hostname, port)
+    end
+
+    def self.get_next_free_host
+      @@hosts.min_by { |h| h.jobs }
+    end
+
+    def self.get_free_hosts()
+      @@hosts.select { |host| host.jobs == 0 }
+    end
+
+    def self.send_to_all(payload)
+      @@hosts.each { |host| host.send payload, false }
+    end
+
+    def self.close_connections
+      @@hosts.map(&:close_connection)
+    end
+
+  end
+end

data/lib/madrox-cluster/json_builder.rb

@@ -0,0 +1,22 @@
+module Madrox
+  class JsonPackage
+
+    def self.register(reference, code)
+      {:type => "register", :reference => reference, :code => code.to_source}.to_json
+    end
+
+    def self.execute(block, args=[])
+      {:type => "execute", :code => block.to_source, :args => args}.to_json
+    end
+
+    def self.result(result)
+      {:type => "result", :result => result}.to_json
+    end
+
+    def self.parse(package)
+      json = JSON.parse(package)
+      OpenStruct.new(json)
+    end
+
+  end
+end

data/lib/madrox-cluster/server/server.rb

@@ -0,0 +1,22 @@
+require 'sourcify'
+require 'socket'
+require 'eventmachine'
+require 'json'
+require 'parallel'
+require 'pry'
+
+require 'madrox-cluster/json_builder'
+require 'madrox-cluster/server/server_handler'
+
+module Madrox
+
+  class Server
+    def self.start(hostname, port=2000)
+      puts "starting server on #{hostname}:#{port}"
+      EventMachine.run {
+        EventMachine.start_server hostname, port, ServerHandler
+      }
+    end
+  end
+
+end

data/lib/madrox-cluster/server/server_handler.rb

@@ -0,0 +1,35 @@
+module Madrox
+  module ServerHandler
+
+    def post_init
+      port, ip = Socket.unpack_sockaddr_in(self.get_peername)
+      puts "#{ip} has connected"
+    end
+
+    def register(reference, code)
+      eval("#{reference} = #{code}.call()")
+    end
+
+    def execute(code, args)
+      args = args.first if args.size == 1
+      result = eval(code).call(args)
+      JsonPackage.result(result)
+    end
+
+    def receive_data(package)
+      Thread.new do
+        data = JsonPackage.parse(package)
+        puts "-----"
+        puts data
+        case data.type
+        when "register"
+          register(data.reference, data.code)
+        when "execute"
+          result = execute(data.code, data.args)
+          send_data result.to_s + "\n"
+        end
+      end
+    end
+
+  end
+end

data/lib/madrox-cluster/version.rb

@@ -0,0 +1,3 @@
+module Madrox
+  VERSION = "0.0.4"
+end

data/lib/madrox-cluster/workers.rb

@@ -0,0 +1,58 @@
+module Madrox
+  class Workers
+
+    def initialize(array, block)
+      @result    = []
+      @job_queue = array.to_a
+      @threads   = []
+      @block     = block
+      @semaphore = Mutex.new
+      @index = -1
+    end
+
+    def execute
+      execute_jobs
+      @threads.map(&:join) #wait for all threads to finish
+      HostsManager.close_connections
+      @result
+    end
+
+    private
+
+    def execute_jobs
+      connections = HostsManager.get_free_hosts
+      initial_jobs = @job_queue.shift connections.count
+
+      initial_jobs.each_with_index do |value, index|
+        @semaphore.lock
+        @index += 1
+        @semaphore.unlock
+        execute_job(connections[index], value, index)
+      end
+    end
+
+    def execute_job(connection, value, index)
+      @threads << Thread.new {
+        response = connection.send JsonPackage.execute(@block, [value])
+        result = JsonPackage.parse(response)
+
+        res = eval(result.result.to_s)
+        @result[index] = res
+        execute_next_job
+      }
+    end
+
+    def execute_next_job
+      @semaphore.lock
+      connection = HostsManager.get_next_free_host
+      value = @job_queue.shift
+      @index += 1
+      index = @index
+      @semaphore.unlock
+
+      return if value.nil?
+      execute_job(connection, value, index)
+    end
+
+  end
+end

data/lib/madrox-cluster.rb

@@ -0,0 +1,47 @@
+require 'sourcify'
+require 'socket'
+require 'eventmachine'
+require 'json'
+require 'parallel'
+require 'pry'
+
+require "madrox-cluster/version"
+require "madrox-cluster/host"
+require "madrox-cluster/hosts_manager"
+require "madrox-cluster/json_builder"
+require "madrox-cluster/workers"
+
+module Madrox
+
+  def self.config(host_list)
+    HostsManager.add_hosts(host_list)
+  end
+
+  def self.register(reference, code)
+    HostsManager.send_to_all JsonPackage.register(reference, code)
+  end
+
+  def self.execute(&block)
+    host = HostsManager.get_next_free_host
+    response = host.send JsonPackage.execute(block)
+    result = JsonPackage.parse(response)
+    eval(result.result.to_s)
+  end
+
+  def self.each(array, options = {}, &block)
+    self.collect(array, options, &block)
+    nil
+  end
+
+  def self.map(array, options={}, &block)
+    self.collect(array, options, &block)
+  end
+
+  def self.collect(array, options = {}, &block)
+    workers = Workers.new(array, block)
+    result  = workers.execute
+    
+    return result
+  end
+
+end

data/madrox-cluster.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "madrox-cluster/version"
+
+Gem::Specification.new do |s|
+  s.name        = "madrox-cluster"
+  s.version     = Madrox::VERSION
+  s.authors     = ["Iuri Matias"]
+  s.email       = ["iuri.matias@gmail.com"]
+  s.homepage    = "https://github.com/iurimatias/madrox-cluster"
+  s.summary     = %q{Straightforward distributed computing in ruby}
+  s.description = %q{Easily distribute any code in multiple servers}
+
+  s.rubyforge_project = "madrox-cluster"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  s.add_development_dependency "pry"
+
+  s.add_runtime_dependency 'ruby_parser'
+  s.add_runtime_dependency 'file-tail' 
+  s.add_runtime_dependency 'sourcify'
+  s.add_runtime_dependency 'eventmachine'
+  s.add_runtime_dependency 'parallel'
+
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification 
+name: madrox-cluster
+version: !ruby/object:Gem::Version 
+  version: 0.0.4
+platform: ruby
+authors: 
+- Iuri Matias
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2013-09-22 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: pry
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - &id002 
+      - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: ruby_parser
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - *id002
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: file-tail
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    requirements: 
+    - *id002
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: sourcify
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    requirements: 
+    - *id002
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    requirements: 
+    - *id002
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: parallel
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    requirements: 
+    - *id002
+  type: :runtime
+  version_requirements: *id007
+description: Easily distribute any code in multiple servers
+email: 
+- iuri.matias@gmail.com
+executables: 
+- madrox
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- Rakefile
+- Readme.md
+- bin/madrox
+- lib/madrox-cluster.rb
+- lib/madrox-cluster/host.rb
+- lib/madrox-cluster/hosts_manager.rb
+- lib/madrox-cluster/json_builder.rb
+- lib/madrox-cluster/server/server.rb
+- lib/madrox-cluster/server/server_handler.rb
+- lib/madrox-cluster/version.rb
+- lib/madrox-cluster/workers.rb
+- madrox-cluster.gemspec
+homepage: https://github.com/iurimatias/madrox-cluster
+licenses: []
+
+metadata: {}
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - *id002
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - *id002
+requirements: []
+
+rubyforge_project: madrox-cluster
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Straightforward distributed computing in ruby
+test_files: []
+