gmalamid-spinal 0.0.0

data/COPYING

@@ -0,0 +1,18 @@
+Copyright (c) 2008 nutrun.com, Andy Kent, George Malamidis
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to
+deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+  
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+   
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,12 @@
+--- It's still very early days and this is experimental code, read: "Here be dragons" ---
+
+Spinal is lightweight middleware for bidirectional process distribution over TCP.
+It comes in three parts all currently implemented in Ruby:
+
+spinald - This is the server daemon, it manages connections and distributes requests. Servers are cluster-able in a share nothing style architecture.
+
+Spinal Client Library - The client Library makes it super simple to delegate work to a Spinal Service, requests can be blocking or asynchronous.
+
+Spinal Service Library - Use the service library to implement workers. As with servers, services are cluster-able and distributed.
+
+See etc/demo for a simplified example using Rack as a client to a directory service.

data/Rakefile

@@ -0,0 +1,16 @@
+require "rubygems"
+require "rake/testtask"
+
+task :default => :test
+
+Rake::TestTask.new do |t|
+  t.pattern = "test/*_test.rb"
+end
+
+task(:check_gemspec) do
+  require 'rubygems/specification'
+  data = File.read('spinal.gemspec')
+  spec = nil
+  Thread.new { spec = eval("$SAFE = 3\n#{data}") }.join
+  puts spec
+end

data/etc/demo/daemon.rb

@@ -0,0 +1,8 @@
+require File.join(File.dirname(__FILE__), '..', '..', 'lib', 'spinal', 'spinald')
+
+Spinal.debug = true
+
+EM.run do
+  trap(:INT) {EM.stop}
+  EM.start_server("localhost", 8888, Spinal::Spinald)
+end

data/etc/demo/web.rb

@@ -0,0 +1,22 @@
+require "rubygems"
+require "thin"
+
+$: << File.join(File.dirname(__FILE__), '..', '..', 'lib')
+
+require "spinal/thin_deferrable_connection"
+require "spinal/rack_remote_query"
+
+Thin::Connection.send(:include, Spinal::ThinDeferrableConnection)
+
+class Webapp
+  include Spinal::Rack::RemoteQuery
+
+  def call(env)
+    request = Rack::Request.new(env)
+    remote_query('wiretap', request.params.to_yaml) do |response|
+      [response.status, {'Content-Type' => 'text/plain'}, response.body]
+    end
+  end
+end
+
+Rack::Handler::Thin.run(Webapp.new, :Host => 'localhost', :Port => 4567)

data/etc/protocol.txt

@@ -0,0 +1,26 @@
+Requests
+
+Query a resource identified by URI
+========================
+QUERY
+address
+ASCII payload
+___END___
+========================
+
+Register Service with Spinal on resource identified by URI 
+========================
+REGISTER
+address
+host:port
+___END___
+========================
+
+
+Responses
+========================
+RESPONSE
+[200|400|...]
+ASCII body
+___END___
+========================

data/lib/spinal/client/blocking.rb

@@ -0,0 +1,28 @@
+require "socket"
+require "fcntl"
+
+$: << File.join(File.dirname(__FILE__), '..')
+
+require "shared"
+
+module Spinal
+  module Client
+    class Blocking
+      def initialize(host, port)
+        @socket = TCPSocket.new(host, port.to_i)
+        @socket.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
+      end
+
+      def query(address, payload)
+        packet = Packet::Query.new(address, payload)
+        @socket.write(packet.raw)
+        packet_parser, resp = PacketParser.new, []
+        loop do
+          str = @socket.gets("\r\n").strip
+          packet_parser.parse(str)
+          return packet_parser.packet if packet_parser.has_packet?
+        end
+      end      
+    end
+  end
+end

data/lib/spinal/client/evented.rb

@@ -0,0 +1,51 @@
+require "rubygems"
+require "eventmachine"
+
+$: << File.join(File.dirname(__FILE__), '..')
+
+require "shared"
+
+module Spinal
+  module Client
+    module Evented
+      def query(uri, payload = '', &response)
+        send_data(Packet::Query.new(uri, payload).raw)
+        @response = response
+      end
+
+      def receive_data(data)
+        packet_parser.parse(data)
+        packet_parser.each_packet { |packet| @response.call(packet) }
+      end
+
+      def packet_parser
+        @packet_parser ||= PacketParser.new
+      end
+
+      def self.connect(opts={})
+        client = nil
+        EM::safe_run(opts[:background]) do
+          client = EM.connect(opts[:host] || '0.0.0.0', opts[:port] || 8888, self)
+        end
+        client
+      end
+    end
+  end
+end
+
+module EventMachine
+  def EventMachine::safe_run(background = nil, &block)
+    if EM::reactor_running?
+      EM::next_tick(&block)
+      sleep unless background
+    else
+      if background
+        $em_reactor_thread = Thread.new do
+          EM::run(&block)
+        end
+      else
+        EM::run(&block)
+      end
+    end
+  end
+end

data/lib/spinal/rack_remote_query.rb

@@ -0,0 +1,35 @@
+module Spinal
+  module Rack
+    module RemoteQuery
+      def remote_query(address, payload='', &response_callback)
+        @response = Spinal::ThinDeferrableConnection::DeferredResponse.new(&response_callback)
+        connection.send_data(Packet::Query.new(address, payload).raw)
+        @response
+      end
+
+      def receive_packet(packet)
+        @response.deliver(packet)
+      end
+
+      def connection
+        @connection ||= EM.connect('0.0.0.0', 8888, SpinalConnection, self)
+      end
+
+      module SpinalConnection
+        def initialize(client)
+          super
+          @client = client
+        end
+
+        def receive_data(data)
+          packet_parser.parse(data)
+          packet_parser.each_packet { |packet| @client.receive_packet(packet) }
+        end
+
+        def packet_parser
+          @packet_parser ||= PacketParser.new
+        end
+      end
+    end
+  end
+end

data/lib/spinal/service.rb

@@ -0,0 +1,29 @@
+require "rubygems"
+require "eventmachine"
+
+$: << File.dirname(__FILE__)
+
+require "shared"
+require "service/registration"
+require "service/connection"
+require "service/runner"
+require "service/request"
+
+module Spinal
+  module Service
+    def call(request)
+      raise "A Spinal::Service must implement the call(request) method"
+    end
+
+    module ClassMethods
+      def run_as(address, opts={})
+        opts[:address] = address
+        Runner.new(self.new, opts).run
+      end
+    end
+
+    def self.included(target)
+      target.extend(ClassMethods)
+    end
+  end
+end

data/lib/spinal/service/connection.rb

@@ -0,0 +1,24 @@
+module Spinal
+  module Service
+    module Connection
+      attr_writer :service, :servers, :address, :host, :port
+    
+      def receive_data(data)
+        packet_parser.parse(data)
+        packet_parser.each_packet do |packet|
+          request = Request.new(@service)
+          request.callback {|response| send_data(response)}
+          request.handle(packet)
+        end
+      end
+
+      def service
+        @service || proc {|req| "No service specified."}
+      end
+
+      def packet_parser
+        @packet_parser ||= PacketParser.new
+      end
+    end
+  end
+end

data/lib/spinal/service/registration.rb

@@ -0,0 +1,34 @@
+module Spinal
+  module Registration
+    def receive_data(data)
+      packet_parser.parse(data)
+      packet_parser.each_packet do |packet|
+        @status = packet.status
+      end
+    end
+    
+    def bind(servers, service_address, service_host, service_port)
+      servers.each do |server|
+        server_host, server_port = server.split(':')
+        conn = EM.connect(server_host, server_port.to_i, self)
+        packet = Packet::Register.new(service_address, service_host, service_port).raw
+        conn.send_data(packet)
+      end
+    end
+    module_function :bind
+    
+    def unbind
+      if @status == StatusCodes::OK
+        puts "Registration complete!"
+      else
+        puts "[WARN] failed to register with spinald"
+      end
+    end
+    
+    private
+
+    def packet_parser
+      @packet_parser ||= PacketParser.new
+    end
+  end
+end

data/lib/spinal/service/request.rb

@@ -0,0 +1,30 @@
+module Spinal
+  module Service
+    class Request
+      include EM::Deferrable
+      attr_reader :service
+
+      def initialize(service)
+        @service = service
+      end
+
+      def handle(packet)
+        @packet = packet
+        EM.spawn do |request|
+          response = begin
+            [StatusCodes::OK, request.service.call(request.payload)]
+          rescue Spinal::Error => e
+            [e.code, e.message]
+          rescue Exception => e
+            [StatusCodes::SERVER_ERROR, e.message]
+          end
+          request.set_deferred_success(Packet::Response.new(*response).raw)
+        end.notify(self)
+      end
+
+      def payload
+        @packet.payload
+      end
+    end
+  end
+end

data/lib/spinal/service/runner.rb

@@ -0,0 +1,45 @@
+module Spinal
+  module Service
+    class Runner
+      DEFAULT_OPTIONS = {
+        :host => "127.0.0.1",
+        :port => 5000,
+        :servers => ["127.0.0.1:8888"]
+      }
+
+      attr_reader :service
+
+      def initialize(service, options)
+        @service = service
+        @options = DEFAULT_OPTIONS.merge(options)
+      end
+
+      def run
+        trap(:INT) {EM.stop}
+        EM.run do
+          EM.start_server(host, port, Spinal::Service::Connection) do |t|
+            t.service,t.host, t.port = service, host, port
+          end
+          puts "#{service} (#{address}) listening on #{host}:#{port}"
+          Registration.bind(servers, address, host, port)
+        end
+      end
+
+      def host
+        @options[:host]
+      end
+
+      def servers
+        @options[:servers]
+      end
+
+      def address
+        @options[:address]
+      end
+      
+      def port
+        @options[:port]
+      end
+    end
+  end
+end

data/lib/spinal/shared.rb

@@ -0,0 +1,6 @@
+$: << File.dirname(__FILE__)
+
+require "shared/packet"
+require "shared/packet_parser"
+require "shared/status_codes"
+require "shared/error"

data/lib/spinal/shared/error.rb

@@ -0,0 +1,12 @@
+module Spinal
+  class Error < Exception
+    attr_reader :code    
+  end
+
+  class NotFound < Error
+    def initialize(message)
+      super
+      @code = Spinal::StatusCodes::NOT_FOUND
+    end
+  end
+end

data/lib/spinal/shared/packet.rb

@@ -0,0 +1,67 @@
+module Spinal
+  module Packet
+    BOUNDARY = "___END___" unless defined?(BOUNDARY)
+    CR = "\r\n" unless defined?(CR)
+
+    def self.from_a(ary)
+      type = ary.shift
+      case type
+      when "QUERY" : Query.from_a(ary)
+      when "RESPONSE" : Response.from_a(ary)
+      when "REGISTER" : Register.from_a(ary)
+      else raise("Unrecognized packet type '#{type}'")
+      end
+    end
+
+    class Query
+      attr_reader :address, :payload
+
+      def initialize(address, payload)
+        @address, @payload = address, payload
+      end
+
+      def self.from_a(ary)
+        self.new(ary.shift, ary.join(CR))
+      end
+
+      def raw
+        ["QUERY", @address, @payload, "#{BOUNDARY}#{CR}"] * CR
+      end
+    end
+
+
+    class Response
+      attr_reader :status, :body
+      
+      def initialize(status, body = '')
+        @status, @body = status.to_i, body
+      end
+      
+      def self.from_a(ary)
+        self.new(ary.shift, ary.join(CR))
+      end
+      
+      def raw
+        ["RESPONSE", @status, @body, "#{BOUNDARY}#{CR}"] * CR
+      end
+    end
+
+    class Register
+      attr_reader :address, :host, :port
+      
+      def initialize(address, host, port)
+        @address, @host, @port = address, host, port.to_i
+      end
+      
+      def self.from_a(ary)
+        address = ary.shift
+        host, port = ary.shift.split(":")
+        self.new(address, host, port)
+      end
+      
+      def raw
+        ["REGISTER", @address, "#{@host}:#{@port}", "#{BOUNDARY}#{CR}"] * CR
+      end
+    end
+  end
+end

data/lib/spinal/shared/packet_parser.rb

@@ -0,0 +1,37 @@
+module Spinal
+  class PacketParser
+    def initialize
+      @packets, @data = [], ''
+    end
+
+    def parse(packet)
+      @data << packet
+      tokens, remainder = [], ''
+      #FIXME: OUCH!! This will iterate through the entire payload/body...
+      @data.split(Packet::CR).each do |token|
+        unless token == Packet::BOUNDARY
+          tokens << token
+          remainder << "#{token}#{Packet::CR}"
+        else
+          @packets << Packet.from_a(tokens)
+          tokens, remainder = [], ''
+        end
+      end
+      @data = remainder
+    end
+
+    def each_packet
+      while packet = @packets.pop
+        yield packet
+      end
+    end
+    
+    def packet
+      @packets.pop
+    end
+    
+    def has_packet?
+      @packets.size == 1
+    end
+  end
+end

data/lib/spinal/shared/status_codes.rb

@@ -0,0 +1,8 @@
+module Spinal
+  module StatusCodes
+    OK = 200
+    NOT_FOUND = 404
+    UNKNOWN_RESOURCE = 418
+    SERVER_ERROR = 500
+  end
+end

data/lib/spinal/spinald.rb

@@ -0,0 +1,118 @@
+require "rubygems"
+require "eventmachine"
+
+$: << File.dirname(__FILE__)
+
+require "shared"
+
+module Spinal
+  #FIXME: Tmp hacks  
+  def debug(str) puts str if @debug end
+  module_function :debug
+  def debug=(bool) @debug = bool end
+  module_function :debug=
+
+  # Keep tmp hacks above this line
+  # ==================================================
+
+  module Spinald
+
+    def receive_data(data)
+      Spinal.debug "Spinal received: #{data}"
+      packet_parser.parse(data)
+      packet_parser.each_packet { |packet| handle_request(packet) }
+    end
+
+    def handle_request(packet)
+      case packet.class.name
+      when 'Spinal::Packet::Query' : handle_query(packet)
+      when 'Spinal::Packet::Register' : handle_registration(packet)
+      else raise("Unknown packet type: #{packet.class.inspect}")
+      end
+    end
+    
+    def handle_query(packet)
+      if resource = resources[packet.address]
+        resource.request(packet.raw) { |resp| send_data(resp) }
+      else
+        send_data(Packet::Response.new(StatusCodes::UNKNOWN_RESOURCE).raw)
+      end
+    end
+    
+    def handle_registration(packet)
+      resources[packet.address] ||= Resource.new(packet.address)
+      resources[packet.address].add_service(packet.host, packet.port)
+      send_data(Packet::Response.new(StatusCodes::OK).raw)
+      close_connection_after_writing
+    end
+
+    def resources
+      @@resources ||= {}
+    end
+
+    def packet_parser
+      @packet_parser ||= PacketParser.new
+    end
+
+    class Resource
+      def initialize(address)
+        @address = address
+        @queue = []
+        @pool = ServiceConnectionPool.new
+      end
+
+      def add_service(host, port)
+        @pool.connect_service(host, port)
+      end
+
+      def request(data, &reply_callback)
+        if conn = @pool.reserve
+          conn.reply &reply_callback
+          conn.release do
+            @pool.release(conn)
+            if queued_request = @queue.pop
+              request(queued_request[0], &queued_request[1])
+            end
+          end
+          conn.send_data(data)
+        else
+          @queue.unshift([data, reply_callback])
+        end
+      end    
+    end
+
+    class ServiceConnectionPool
+      def initialize
+        @connections = []
+      end
+
+      def connect_service(host, port)
+        @connections.unshift(EM.connect(host, port, ServiceConnection))
+      end
+
+      def reserve
+        @connections.pop
+      end
+
+      def release(conn)
+        @connections.unshift(conn)
+      end
+    end
+
+    module ServiceConnection
+      def receive_data(data)
+        Spinal.debug "ServiceConnection received: #{data}"
+        @reply.call(data)
+        @release.call
+      end
+
+      def reply(&cb)
+        @reply = cb
+      end
+
+      def release(&cb)
+        @release = cb
+      end
+    end
+  end
+end

data/lib/spinal/thin_deferrable_connection.rb

@@ -0,0 +1,40 @@
+$: << File.dirname(__FILE__)
+require "shared"
+
+module Spinal
+  module ThinDeferrableConnection
+    def process
+      if threaded?
+        @request.threaded = true
+        EventMachine.defer(method(:pre_process), method(:post_process))
+      else
+        @request.threaded = false
+        pre_process_result = pre_process
+        if pre_process_result.is_a?(DeferredResponse)
+          pre_process_result.post_process_callback { |res| post_process(res) }
+        else
+          post_process(pre_process_result)
+        end
+      end
+    end
+
+    def self.included(target)
+      target.send(:remove_method, :process)
+    end
+
+    class DeferredResponse
+      def initialize(&response_callback)
+        @response_callback = response_callback
+      end
+
+      def post_process_callback(&post_process_callback)
+        @post_process_callback = post_process_callback
+      end
+
+      def deliver(packet)
+        result = @response_callback.call(packet)
+        @post_process_callback.call(result)
+      end
+    end
+  end
+end

data/spinal.gemspec

@@ -0,0 +1,38 @@
+GEMSPEC =Gem::Specification.new do |s|
+  s.name = 'spinal'
+  s.version = '0.0.0'
+  s.platform = Gem::Platform::RUBY
+  s.rubyforge_project = "spinal"
+  s.summary, s.description = 'Lightweight middleware for bidirectional process distribution over TCP'
+  s.authors = 'Andy Kent, George Malamidis'
+  s.email = 'george@nutrun.com'
+  # s.homepage = 'http://spinal.rubyforge.org'
+  s.has_rdoc = false
+  s.rdoc_options += ['--quiet', '--title', 'Spinal', '--main', 'README']
+  s.extra_rdoc_files = ['README', 'COPYING']
+  s.files = [
+    "COPYING",
+    "Rakefile",
+    "README",
+    "spinal.gemspec",
+    "etc/demo/daemon.rb",
+    "etc/demo/service.rb",
+    "etc/demo/web.rb",
+    "etc/protocol.txt",
+    "lib/spinal/client/blocking.rb",
+    "lib/spinal/client/evented.rb",
+    "lib/spinal/rack_remote_query.rb",
+    "lib/spinal/service/connection.rb",
+    "lib/spinal/service/registration.rb",
+    "lib/spinal/service/request.rb",
+    "lib/spinal/service/runner.rb",
+    "lib/spinal/service.rb",
+    "lib/spinal/shared/error.rb",
+    "lib/spinal/shared/packet.rb",
+    "lib/spinal/shared/packet_parser.rb",
+    "lib/spinal/shared/status_codes.rb",
+    "lib/spinal/shared.rb",
+    "lib/spinal/spinald.rb",
+    "lib/spinal/thin_deferrable_connection.rb"
+  ]
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification 
+name: gmalamid-spinal
+version: !ruby/object:Gem::Version 
+  version: 0.0.0
+platform: ruby
+authors: 
+- Andy Kent, George Malamidis
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-10-27 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: 
+email: george@nutrun.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README
+- COPYING
+files: 
+- COPYING
+- Rakefile
+- README
+- spinal.gemspec
+- etc/demo/daemon.rb
+- etc/demo/service.rb
+- etc/demo/web.rb
+- etc/protocol.txt
+- lib/spinal/client/blocking.rb
+- lib/spinal/client/evented.rb
+- lib/spinal/rack_remote_query.rb
+- lib/spinal/service/connection.rb
+- lib/spinal/service/registration.rb
+- lib/spinal/service/request.rb
+- lib/spinal/service/runner.rb
+- lib/spinal/service.rb
+- lib/spinal/shared/error.rb
+- lib/spinal/shared/packet.rb
+- lib/spinal/shared/packet_parser.rb
+- lib/spinal/shared/status_codes.rb
+- lib/spinal/shared.rb
+- lib/spinal/spinald.rb
+- lib/spinal/thin_deferrable_connection.rb
+has_rdoc: false
+homepage: 
+post_install_message: 
+rdoc_options: 
+- --quiet
+- --title
+- Spinal
+- --main
+- README
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: spinal
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Lightweight middleware for bidirectional process distribution over TCP
+test_files: []
+