cod 0.3.1 → 0.4.0

data/Gemfile

@@ -1,6 +1,6 @@
 source "http://rubygems.org"
 
-gem 'uuid'
+gem 'rake'
 gem 'beanstalk-client'
 
 group :test do

data/HISTORY.txt

@@ -1,5 +1,9 @@
+== 0.4.0 / 29Nov2011
 
-== 0.3.1 / ???
+  * A complete rewrite, focusing on lean implementation and logical/useful 
+    error conditions. 
+
+== 0.3.1 / 25Aug2011
 
   + Large improvements in resilience to crashes, reconnects are now attempted
     but can be improved further. 

data/README

@@ -1,33 +1,32 @@
 
-cod is a simple ipc abstraction layer. It allows you to focus on interaction 
-between processes instead of having to think about interaction with the OS. 
+IPC for babies and those who want to become. Another thin API layer over what
+Ruby offers for IO.pipe, TCP sockets and FIFOs: Makes sending and receiving
+of Ruby objects a breeze. 
+
+A good place to start is the documentation for the Cod module.
 
 SYNOPSIS
 
   # Cod's basic elements are channels, unidirectional communication links. 
   pipe = Cod.pipe
-  beanstalk = Cod.beanstalk('localhost:11300', 'a_channel')
   
   # You can use those either directly: 
   pipe.put :some_ruby_object        # Process A
   pipe.get # => :some_ruby_object   # Process B
   
   # Or use them as bricks for more: 
-  service = Cod::Service.new(beanstalk)
-  client = Cod::Client.new(beanstalk, pipe)
+  service = beanstalk.service
+  client  = beanstalk.client(pipe)
   
   service.one { |msg| :response }               # Process A
   client.call :ruby_object    # => :response    # Process B
-  
-  # And more: Publish/Subscribe, easy construction of more advanced
-  # distributed communication.
-  
+    
 STATUS
 
-Becoming more useful by the day. Most things will work nicely already,
-although error handling is not production quality. Toy around with it now
-and give me feedback!
+Complete rewrite of the code: Did away with some of the complexities that
+stemmed from early design work. The functionality that is there is much like
+that we had before, but some things have been designed more logically. 
 
-At version 0.3.1
+At version 0.4.0
 
 (c) 2011 Kaspar Schiess

data/Rakefile

@@ -1,4 +1,3 @@
-require 'psych'
 require "rubygems"
 require "rdoc/task"
 require 'rspec/core/rake_task'
@@ -9,6 +8,13 @@ RSpec::Core::RakeTask.new
 
 task :default => :spec
 
+task :stats do
+  %w(lib spec).each do |path|
+    printf "%10s:", path
+    system %Q(find #{path} -name "*.rb" | xargs wc -l | grep total)
+  end
+end
+
 require 'sdoc'
 
 # Generate documentation

data/examples/{ping.rb → ping_pong/ping.rb}

@@ -1,4 +1,4 @@
-$:.unshift File.expand_path(File.dirname(__FILE__) + "/../lib")
+$:.unshift File.expand_path(File.dirname(__FILE__) + "/../../lib")
 require 'cod'
 
 pipe = Cod.beanstalk('localhost:11300', 'pingpong')

data/examples/{pong.rb → ping_pong/pong.rb}

@@ -1,4 +1,4 @@
-$:.unshift File.expand_path(File.dirname(__FILE__) + "/../lib")
+$:.unshift File.expand_path(File.dirname(__FILE__) + "/../../lib")
 require 'cod'
 
 pipe = Cod.beanstalk('localhost:11300', "pingpong")

data/examples/queue/README

@@ -0,0 +1,9 @@
+A small queue like example. 
+
+run ruby send.rb N to send N messages through queue.rb to client.rb. 
+Client will print statistics (calls per second) every 100 messages.
+
+Problems: 
+  - queue.rb seems to always consume 100% CPU
+  - number of messages per second is way low
+  - some exceptions are still not handled correctly 

data/examples/queue/client.rb

@@ -0,0 +1,31 @@
+$:.unshift File.expand_path(File.dirname(__FILE__) + "/../../lib")
+require 'cod'
+
+queue = Cod.tcp('localhost:12345')
+
+queue.put [:join, queue]
+
+class CallCounter
+  attr_reader :n
+  def initialize
+    @n = 0
+    @last_lap = [0, Time.now]
+  end
+  def inc
+    @n += 1
+  end
+  def calls_per_sec
+    ln, ll = @last_lap
+    @last_lap = [@n, Time.now]
+    
+    (@n - ln) / (Time.now - ll)
+  end
+end
+
+cc = CallCounter.new
+loop do
+  wi = queue.get
+  
+  cc.inc
+  puts cc.calls_per_sec if cc.n%100==0 
+end

data/examples/queue/queue.rb

@@ -0,0 +1,51 @@
+$:.unshift File.expand_path(File.dirname(__FILE__) + "/../../lib")
+require 'cod'
+
+class Queue
+  def initialize(url)
+    @url = url
+    connect
+  end
+  
+  def connect
+    @clients = []
+    @server = Cod.tcpserver(@url)
+  end
+  
+  def run
+    loop do
+      handle_commands
+      
+      check_connections
+    end
+  end
+  
+  def handle_commands
+    while @server.waiting?
+      cmd, *rest = @server.get
+      
+      dispatch_command cmd, rest
+    end
+  end
+  
+  def dispatch_command(cmd, rest)
+    self.send("cmd_#{cmd}", *rest)
+  end
+  
+  def cmd_join(connection)
+    puts "Join at #{Time.now}."
+    @clients << connection
+  end
+  
+  def cmd_work_item
+    @clients.each do |client|
+      client.put :work_item
+    end
+  end
+  
+  def check_connections
+    @clients.keep_if { |client| client.connected? }
+  end
+end
+
+Queue.new('localhost:12345').run

data/examples/queue/send.rb

@@ -0,0 +1,9 @@
+$:.unshift File.expand_path(File.dirname(__FILE__) + "/../../lib")
+require 'cod'
+
+queue = Cod.tcp('localhost:12345')
+
+n = Integer(ARGV.first)
+n.times do
+  queue.put :work_item
+end

data/examples/service.rb

@@ -9,7 +9,7 @@ service_channel = Cod.pipe
 answer_channel = Cod.pipe
 
 child_pid = fork do
-  service = Cod::Service.new(service_channel)
+  service = Cod.service(service_channel)
   service.one { |call| 
     puts "Service got called with #{call.inspect}" 
     time = Time.now
@@ -17,7 +17,7 @@ child_pid = fork do
     time }
 end
 
-client = Cod::Client.new(service_channel, answer_channel)
+client = Cod.client(service_channel, answer_channel)
 puts "Calling service..."
 answer = client.call('42')
 

data/examples/tcp.rb

@@ -5,7 +5,7 @@ require 'cod'
 require 'example_scaffold'
 
 server {
-  channel = Cod.tcpserver('127.0.0.1:5454')
+  channel = Cod.tcp_server('127.0.0.1:5454')
   
   client = channel.get
   client.put 'heiho from server'

data/lib/cod.rb

@@ -1,5 +1,3 @@
-require 'uuid'
-
 # The core concept of Cod are 'channels'. (Cod::Channel::Base) You can create
 # such channels on top of the various transport layers. Once you have such a
 # channel, you #put messages into it and you #get messages out of it. Messages
@@ -8,10 +6,7 @@ require 'uuid'
 #
 # Cod also brings a few abstractions layered on top of channels: You can use
 # channels to present 'services' (Cod::Service) to the network: A service is a
-# simple one or two way RPC call. (one way = asynchronous) You can also use
-# channels to run a 'directory' (Cod::Directory) where processes subscribe to
-# information using a filter. They then get information that matches their
-# filter written to their inbound channel. (also called pub/sub)
+# simple one or two way RPC call. (one way = asynchronous) 
 #
 # Cod channels are serializable whereever possible. If you want to tell
 # somebody where to write his answers and/or questions to, send him the
@@ -28,101 +23,71 @@ require 'uuid'
 # most of the tricky stuff!
 #
 module Cod
-  # This gets raised in #create_reference when the identifier passed in is
-  # either invalid (has never existed) or when it cannot be turned into an
-  # object instance. (Because it might have been garbage collected or other
-  # such reasons)
-  #
-  class InvalidIdentifier < StandardError; end
-  
-  # Creates a beanstalkd based channel. Messages are written to a tube and 
-  # read from it. This channel is read/write. Multiple readers will obtain
-  # messages in a round-robin fashion from the beanstalk server. 
-  #
-  # Returns an instance of the Cod::Channel::Beanstalk class.
+  # Creates a pipe connection that is visible to this process and its
+  # children. (see Cod::Pipe)
   #
-  # Example: 
-  #   chan = Cod.beanstalk('localhost:11300', 'my_tube')
-  #
-  def beanstalk(url, name)
-    Cod::Channel::Beanstalk.new(
-      Connection::Beanstalk.new(url), 
-      name)
-  end
-  module_function :beanstalk
-  
-  # Creates a IO.pipe based channel. Messages are written to one end of the 
-  # pipe and come out on the other end. This channel can have only one reader, 
-  # but of course multiple writers. Also, once you either write or read from 
-  # such a channel, it will not be available for the other operation anymore. 
-  #
-  # A common trick is to #dup the channel before using it to either read or
-  # write, so that the copy can still be used for both operations. 
-  #
-  # Note that Cod.pipe channels are usable from process childs (#fork) as 
-  # well. As such, they are ideally suited for process control. 
-  #
-  # Returns an instance of the Cod::Channel::Pipe class.
-  #
-  # Example: 
-  #   chan = Cod.pipe
-  # 
-  def pipe(name=nil)
-    Cod::Channel::Pipe.new(name)
+  def pipe
+    Cod::Pipe.new
   end
   module_function :pipe
   
-  # Creates a tcp connection to the destination and returns a channel for it. 
+  # Creates a tcp connection to the destination and returns a channel for it.
+  # (see Cod::TcpClient)
   #
-  def tcp(destination)
-    Cod::Channel::TCPConnection.new(destination)
+  def tcp(destination, serializer=nil)
+    Cod::TcpClient.new(
+      destination, 
+      serializer || SimpleSerializer.new)
   end
   module_function :tcp
   
-  # Creates a tcp listener on bind_to and returns a channel for it. 
+  # Creates a tcp listener on bind_to and returns a channel for it. (see
+  # Cod::TcpServer)
   #
-  def tcpserver(bind_to)
-    Cod::Channel::TCPServer.new(bind_to)
+  def tcp_server(bind_to)
+    Cod::TcpServer.new(bind_to)
   end
-  module_function :tcpserver
-  
-  # Returns a UUID that should be unique for this machine (based on MAC), this 
-  # Thread and Process. Even after a fork. 
+  module_function :tcp_server
+
+  # Creates a channel based on the beanstalkd messaging queue. (see
+  # Cod::Beanstalk::Channel)
   # 
-  # This is used to create identity on the network. Internal method. 
+  def beanstalk(tube_name, server=nil)
+    Cod::Beanstalk::Channel.new(tube_name, server||'localhost:11300')
+  end
+  module_function :beanstalk
+
+  # Indicates that the given channel is write only. This gets raised on 
+  # operations like #put.
+  #
+  class WriteOnlyChannel < StandardError
+    def initialize
+      super("This channel is write only, attempted read operation.")
+    end
+  end
+  
+  # Indicates that the channel is read only. This gets raised on operations
+  # like #get. 
   #
-  def uuid
-    uuid_generator.generate
-  end  
-  def uuid_generator
-    pid, generator = Thread.current[:_cod_uuid_generator]
-    
-    if pid && Process.pid == pid
-      return generator
+  class ReadOnlyChannel < StandardError
+    def initialize
+      super("This channel is read only, attempted write operation.")
     end
-    
-    pid, generator = Thread.current[:_cod_uuid_generator] = [Process.pid, UUID.new]
-    return generator
   end
-  module_function :uuid, :uuid_generator
 end
 
-module Cod::Connection; end
-require 'cod/connection/beanstalk'
-
-require 'cod/object_io'
+require 'cod/select_group'
+require 'cod/select'
 
 require 'cod/channel'
-require 'cod/channel/base'
-require 'cod/channel/pipe'
-require 'cod/channel/beanstalk'
-require 'cod/channel/tcpconnection'
-require 'cod/channel/tcpserver'
 
-require 'cod/client'
+require 'cod/simple_serializer'
+
+require 'cod/pipe'
+
+require 'cod/tcp_client'
+require 'cod/tcp_server'
 
 require 'cod/service'
+require 'cod/beanstalk'
 
-require 'cod/directory'
-require 'cod/directory/subscription'
-require 'cod/topic'

data/lib/cod/beanstalk.rb

@@ -0,0 +1,7 @@
+module Cod::Beanstalk
+end
+
+require 'cod/beanstalk/serializer'
+require 'cod/beanstalk/channel'
+require 'cod/beanstalk/service'
+

data/lib/cod/beanstalk/channel.rb

@@ -0,0 +1,170 @@
+module Cod::Beanstalk
+
+  # NOTE: Beanstalk channels cannot currently be used in Cod.select. This is 
+  # due to limitations inherent in the beanstalkd protocol. We'll probably 
+  # try to get a patch into beanstalkd to change this. 
+  #
+  # NOTE: If you embed a beanstalk channel into one of your messages, you will
+  # get a channel that connects to the same server and the same tube on the
+  # other end. This behaviour is useful for Cod::Service.
+  #
+  class Channel < Cod::Channel
+    JOB_PRIORITY = 0
+    
+    def initialize(tube_name, server_url)
+      @tube_name, @server_url = tube_name, server_url
+    
+      @body_serializer = Cod::SimpleSerializer.new
+      @transport = connection(server_url, tube_name)
+    end
+    
+    def put(msg)
+      pri   = JOB_PRIORITY
+      delay = 0
+      ttr   = 120
+      body = @body_serializer.en(msg)
+      
+      answer, *rest = @transport.interact([:put, pri, delay, ttr, body])
+      fail "#put fails, #{answer.inspect}" unless answer == :inserted
+    end
+  
+    def get
+      id, msg = bs_reserve
+      
+      # We delete the job immediately, since #get should be definitive.
+      bs_delete(id)
+
+      deserialize(msg)
+    end
+  
+    def close
+      @transport.close
+    end
+    
+    def to_read_fds
+      fail "Cod.select not supported with beanstalkd channels.\n"+
+        "To support this, we will have to extend the beanstalkd protocol."
+    end
+    
+    # --------------------------------------------------------- service/client
+    def service
+      Service.new(self)
+    end
+    def client(answers_to)
+      Service::Client.new(self, answers_to)
+    end
+    
+    # -------------------------------------------------------- queue interface     
+    def try_get 
+      fail "No block given to #try_get" unless block_given?
+      
+      id, msg = bs_reserve
+      control = Control.new(id, self)
+      
+      begin
+        retval = yield(deserialize(msg), control)
+      rescue Exception
+        control.release unless control.command_given?
+        raise
+      ensure
+        control.delete unless control.command_given?
+      end
+      
+      return retval
+    end
+    
+    # Holds a message id of a reserved message. Allows several commands to be
+    # executed on the message. See #try_get.
+    class Control # :nodoc:
+      attr_reader :msg_id
+      
+      def initialize(msg_id, channel)
+        @msg_id = msg_id
+        @channel = channel
+        @command_given = false
+      end
+        
+      def command_given?
+        @command_given
+      end
+      
+      def delete
+        @command_given = true
+        @channel.bs_delete(@msg_id)
+      end
+      def release
+        @command_given = true
+        @channel.bs_release(@msg_id)
+      end
+      def release_with_delay(seconds)
+        fail ArgumentError, "Only integer number of seconds are allowed." \
+          unless seconds.floor == seconds
+        
+        @command_given = true
+        @channel.bs_release_with_delay(@msg_id, seconds)
+      end
+      def bury
+        @command_given = true
+        @channel.bs_bury(@msg_id)
+      end
+    end
+        
+    # ---------------------------------------------------------- serialization
+    def _dump(level) # :nodoc:
+      Marshal.dump(
+        [@tube_name, @server_url])
+    end
+    def self._load(str) # :nodoc:
+      tube_name, server_url = Marshal.load(str)
+      Cod.beanstalk(tube_name, server_url)
+    end
+
+    # ----------------------------------------------------- beanstalk commands
+    def bs_delete(msg_id) # :nodoc:
+      bs_command([:delete, msg_id], :deleted)
+    end
+    def bs_release(msg_id) # :nodoc:
+      bs_command([:release, msg_id, JOB_PRIORITY, 0], :released)
+    end
+    def bs_release_with_delay(msg_id, seconds) # :nodoc:
+      bs_command([:release, msg_id, JOB_PRIORITY, seconds], :released)
+    end
+    def bs_bury(msg_id)
+      # NOTE: Why I need to assign a priority when burying I fail to
+      # understand. Like a priority for rapture?
+      bs_command([:bury, msg_id, JOB_PRIORITY], :buried)
+    end
+  private 
+    def bs_reserve
+      answer, *rest = bs_command([:reserve], :reserved)
+      rest
+    end
+    def bs_command(cmd, good_answer)
+      answer, *rest = @transport.interact(cmd)
+      fail "#{cmd.first.inspect} fails, #{answer.inspect}" \
+        unless answer == good_answer
+      [answer, *rest]
+    end
+    
+    def deserialize(msg)
+      @body_serializer.de(StringIO.new(msg))
+    end
+  
+    def connection(server_url, tube_name)
+      conn = Cod.tcp(server_url, Serializer.new)
+
+      begin
+        answer, *rest = conn.interact([:use, tube_name])
+        fail "#init_tube fails, #{answer.inspect}" unless answer == :using
+      
+        answer, *rest = conn.interact([:watch, tube_name])
+        fail "#init_tube fails, #{answer.inspect}" unless answer == :watching
+      rescue 
+        conn.close
+        raise
+      end
+      
+      conn
+    end
+  end
+end