stormy-cloud 0.0.8

data/lib/transport.rb

@@ -0,0 +1,269 @@
+require 'base64'
+require 'digest'
+require 'msgpack'
+require 'securerandom'
+require 'set'
+require 'thread'
+
+# Define the basic outline of a transport, and provide serialization and stuff
+# so that code doesn't have to be duplicated in every transport definition.
+class StormyCloudTransport
+  attr_reader :secret, :stormy_cloud, :queue, :task_count
+  attr_accessor :mode
+
+  # This method should be "extended" by the specific transports, i.e. they
+  # should first call super and then perform any transport-specific
+  # instantiation.
+  def initialize(stormy_cloud)
+    # Generate a secret that will be used to shutdown the server.
+    @secret       = SecureRandom.hex(32)
+    # A hash of identifier -> time of last server access.
+    @clients      = {}
+    # Are we operating in server mode or client mode?
+    @mode         = :server
+    # Save `stormy_cloud`.
+    @stormy_cloud = stormy_cloud
+    # Create a new empty queue for storing sub tasks.
+    @queue        = Queue.new
+    # A list of tasks that are currently being worked on.
+    @assigned     = []
+    # A mutex to synchronize access to the queue, assigned list and completed
+    # list.
+    @queue_mutex  = Mutex.new
+    # A set of completed tasks.
+    # TODO: This needs to be synchronized with a file for persistence.
+    @completed    = Set.new
+    # Statistics.
+    @task_count = 0
+    @completed_count = 0
+  end
+
+  # A unique identifier derived from the secret.
+  def identifier
+    Digest::MD5.hexdigest("Omikron" + @secret + "9861")
+  end
+
+  # Run the split method on the stormy cloud, and save the results into a
+  # queue.
+  def split
+    tasks = @stormy_cloud.split
+    @task_count = tasks.length
+    tasks.each {|x| @queue.push x }
+  end
+
+  # Remove tasks from the queue until we encounter one which is not on the
+  # completed list. Add this task to the assigned list, and spawn a thread
+  # to move it back into the queue after a timeout.
+  # If the queue is empty, return a random task from the assigned list.
+  # If both the queue and the assigned list are empty, return nil.
+  def get_task
+    @queue_mutex.synchronize do
+      if @queue.empty?
+
+        if @assigned.length == 0
+          return nil
+        else
+          task = @assigned.sample
+          _spawn_remover(task)
+          return task
+        end
+
+      else
+
+        task = @queue.pop
+        if @completed.include? task
+          return get
+        else
+          _spawn_remover(task)
+          @assigned.push task
+          return task
+        end
+
+      end
+    end
+  end
+
+  # Spawn a thread that will wait for some time and then remove the task from
+  # the assigned list and add it back to the queue if it is still in the 
+  # assigned list.
+  def _spawn_remover(task)
+    Thread.new do
+      sleep @stormy_cloud.config(:wait)
+      @queue_mutex.synchronize do
+        if @assigned.include? task
+          @assigned.delete(task)
+          @queue.push(task)
+        end
+      end
+    end
+  end
+
+  # Accept the results of a task from a node. If the task is still in the
+  # assigned list, get it out of there, put it in the completed set and
+  # call the reduce method.
+  # If the task is not on the assigned list and is also not on the completed
+  # list, add it to the completed list and call the reduce method.
+  # If the task is already in the completed set, do nothing.
+  # If the job is complete, call finally.
+  def put_task(task, result)
+    @queue_mutex.synchronize do
+      return if @completed.include? task
+
+      if @assigned.include? task
+        @assigned.delete task
+      end
+
+      if not @completed.include? task
+        @completed.add task
+        @stormy_cloud.reduce(task, result)
+        @completed_count += 1
+      end
+    end
+
+    if complete?
+      @stormy_cloud.finally
+    end
+  end
+
+  # Is the job complete? This is true if the queue is empty and the assigned
+  # list is empty.
+  def complete?
+    @queue.empty? and @assigned.empty?
+  end
+
+  # This method is used by the server to handle communication with clients.
+  # It should not be overridden by specific transports. It accepts a string
+  # which is a serialized command sent by the server, and returns another
+  # serialized string which is the response that should be sent to the client.
+  # All "commands" are basically arrays in which the first element is the
+  # action and subsequent elements are parameters. _Every_ command will have
+  # the identifier as its first parameter.
+  #
+  # The following actions are supported:
+  #   HELLO(identifier)         -> get the server's identifier.
+  #   GET(identifier)           -> get a new task from the server.
+  #   PUT(identifier, task, result)
+  #                             -> return the result of a task to the server.
+  #
+  def handle(string)
+    valid_commands = ["HELLO", "GET", "PUT"]
+    command = unserialize(string)
+    
+    if not (command.kind_of?(Array) and valid_commands.include? command[0])
+      # The command is invalid.
+      return serialize("INVALID COMMAND")
+    end
+
+    # Update the time of last access.
+    @clients[command[1]] = Time.now
+
+    if command[0] == "HELLO"
+
+      if command.length == 2
+        return serialize(identifier)
+      else
+        return serialize("INVALID COMMAND")
+      end
+
+    elsif command[0] == "GET"
+
+      return serialize(get_task)
+
+    elsif command[0] == "PUT"
+
+      task = command[2]
+      result = command[3]
+
+      if command.length != 4
+        return serialize("INVALID COMMAND")
+      else
+        put_task(task, result)
+        return serialize("OKAY")
+      end
+      
+    end
+  end
+
+  # Initialize a server that will accept commands from nodes and forward them
+  # to the handle method. The server should operate in a loop that is broken
+  # when @mode is no longer :server. This method should be nonblocking.
+  # This method should be implemented by the specific transport.
+  def initialize_server
+    raise NotImplementedError
+  end
+
+  # The server should be killed when this method is called.
+  # This method should be implemented by the specific transport.
+  def kill_server
+    raise NotImplementedError
+  end
+
+  # A wrapper around the raw_send_message method that does serialization and
+  # unserialization.
+  def send_message(object)
+    unserialize(raw_send_message(serialize(object)))
+  end
+
+  # Send a raw string to the server, and return the response. 
+  # This method should be implemented by the specific transport.
+  def raw_send_message(string)
+    raise NotImplementedError
+  end
+
+  # Serialize a request into a string by doing MsgPack + Base64 encoding.
+  # This can be overridden by the specific transport if the protocol used is
+  # such that this mode of serialization is disadvantageous.
+  def serialize(object)
+    Base64::encode64(object.to_msgpack)
+  end
+
+  # Unserialize an object which has been serialized using the `serialize`
+  # method.
+  def unserialize(string)
+    MessagePack.unpack(Base64::decode64(string))
+  end
+
+  # Return all variables relevant to the job status.
+  def status
+    {
+      clients: @clients,
+      assigned: @assigned,
+      task_count: @task_count,
+      completed_count: @completed_count,
+      assigned_count: @assigned.length,
+      name: @stormy_cloud.name
+    }
+  end
+
+  # Block until the job is complete -- meant for usage by the server.
+  def block_until_complete
+    while not complete?
+      sleep 1
+    end
+  end
+
+  def result
+    @stormy_cloud.result
+  end
+
+  # Actually run the task.
+  # First, we check whether we are currently the server by asking the server
+  # for its identifier and seeing if it matches ours.
+  def run
+    if @mode == :server
+      split
+      initialize_server
+      block_until_complete
+    else
+      @mode = :client
+      # Do clienty things.
+      loop do
+        task = send_message(["GET", identifier])
+        break if task == nil
+        result = @stormy_cloud.map(task)
+        send_message(["PUT", identifier, task, result])
+      end
+    end
+  end
+end
+

data/lib/transports/tcp.rb

@@ -0,0 +1,40 @@
+require_relative "../transport.rb"
+require 'socket'
+
+class StormyCloudTCPTransport < StormyCloudTransport
+  def initialize_server
+    @server = TCPServer.new @stormy_cloud.config(:port)
+    @server_thread = Thread.new do
+      loop do
+        Thread.start(@server.accept) do |client|
+          client.puts handle(client.gets)
+        end
+      end
+    end
+  end
+
+  def kill_server
+    @server_thread.kill
+  end
+
+  def raw_send_message(string)
+    begin
+      s = TCPSocket.new(@stormy_cloud.server, @stormy_cloud.config(:port))
+      s.puts string
+      res = s.gets
+      s.close
+      res
+    rescue
+      @errors ||= 0
+      @errors += 1
+      if @errors <= 5
+        STDERR.puts "An error occurred while contacting the server, trying again."
+        sleep 1
+        raw_send_message(string)
+      else
+        STDERR.puts "Could not contact the server, exiting."
+        exit
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,58 @@
+--- !ruby/object:Gem::Specification
+name: stormy-cloud
+version: !ruby/object:Gem::Version
+  version: 0.0.8
+  prerelease: 
+platform: ruby
+authors:
+- Vikhyat Korrapati
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-06-18 00:00:00.000000000 Z
+dependencies: []
+description: StormyCloud makes writing distributed applications in Ruby a piece of
+  cake.
+email: c@vikhyat.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/stormy-cloud.rb
+- lib/transport.rb
+- lib/transports/tcp.rb
+- dashboard/dash.rb
+- dashboard/index.html
+- dashboard/public/images/apple-touch-icon-114x114.png
+- dashboard/public/images/apple-touch-icon-72x72.png
+- dashboard/public/images/apple-touch-icon.png
+- dashboard/public/images/favicon.ico
+- dashboard/public/jquery.js
+- dashboard/public/stylesheets/base.css
+- dashboard/public/stylesheets/layout.css
+- dashboard/public/stylesheets/skeleton.css
+homepage: https://github.com/vikhyat/StormyCloud
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Ridiculously simple distributed applications.
+test_files: []