process_shared 0.0.3 → 0.0.4

data/README.rdoc

@@ -59,12 +59,35 @@ Install the gem with:
 
     puts "value should be zero: #{mem.get_int(0)}"
 
+== Transfer Objects Across Processes
+
+    # allocate a sufficient memory block
+    mem = ProcessShared::SharedMemory.new(1024)
+
+    # sub process can write (serialize) object to memory (with bounds checking)
+    pid = fork do
+      mem.write_object(['a', 'b'])
+    end
+
+    Process.wait(pid)
+
+    # parent process can read the object back (synchronizing access
+    # with a Mutex left as an excercie to reader)
+
+    mem.read_object.must_equal ['a', 'b']
+
 == Todo
 
-* Implement ConditionVariable
+* Test ConditionVariable
 * Implement optional override of core Thread/Mutex classes
 * Extend libpsem to win32?  (See Python's processing library)
 * Break out tests that use PSem.getvalue() (which isn't supported on Mac OS X)
   so that the test suite will pass
 * Add finalizer to Mutex? (finalizer on Semaphore objects may be enough) or a method to
-  explicitly close and release resources?
+  explicitly close and release resources?
+* Test semantics of crashing processes who still hold locks, etc.
+* Is SharedArray with Enumerable mixing sufficient Array-like interface?
+* Remove bsem from libpsem as it is of little use and doesn't work on Mac OS X
+* Possibly implement BoundedSemaphore with arbitrary bound (in Ruby
+  rather than relying on sem_getvalue()), but this is of little
+  utility beyond extra error checking..

data/ext/libpsem/psem_posix.c

@@ -68,7 +68,6 @@ psem_free(psem_t *psem) {
       error_new((err), E_SOURCE_SYSTEM, errno);		\
       return ERROR;					\
     }							\
-    return OK;						\
   } while (0)
 
 #define errcheck(expr, err) errcheck_val((expr), -1, (err))
@@ -79,52 +78,83 @@ psem_open(psem_t *psem, const char *name, unsigned int value, error_t **err)
   errcheck_val(psem->sem = sem_open(name, O_CREAT | O_EXCL, 0600, value),
 	       SEM_FAILED,
 	       err);
+  return OK;
 }
 
 int
 psem_close(psem_t *psem, error_t **err)
 {
   errcheck(sem_close(psem->sem), err);
+  return OK;
 }
 
 int
 psem_unlink(const char *name, error_t **err)
 {
   errcheck(sem_unlink(name), err);
+  return OK;
 }
 
 int
 psem_post(psem_t *psem, error_t **err)
 {
   errcheck(sem_post(psem->sem), err);
+  return OK;
 }
 
 int
 psem_wait(psem_t *psem, error_t **err)
 {
   errcheck(sem_wait(psem->sem), err);
+  return OK;
 }
 
 int
 psem_trywait(psem_t *psem, error_t **err)
 {
   errcheck(sem_trywait(psem->sem), err);
+  return OK;
 }
 
+#define NS_PER_S (1000 * 1000 * 1000)
+#define US_PER_NS (1000)
+#define TV_NSEC_MAX (NS_PER_S - 1)
+
 int
 psem_timedwait(psem_t *psem, float timeout_s, error_t **err)
 {
+  struct timeval now;
   struct timespec abs_timeout;
 
-  abs_timeout.tv_sec = floorf(timeout_s);
-  abs_timeout.tv_nsec =
-    floorf((timeout_s - abs_timeout.tv_sec) * (1000 * 1000 * 1000));
+  errcheck(gettimeofday(&now, NULL), err);
+  abs_timeout.tv_sec = now.tv_sec;
+  abs_timeout.tv_nsec = now.tv_usec * US_PER_NS;
+
+  /* Fun with rounding: careful adding reltive timeout to abs time */
+  {
+    time_t sec;		/* relative timeout */
+    long nsec;
+  
+    sec = floorf(timeout_s);
+    nsec = floorf((timeout_s - floorf(timeout_s)) * NS_PER_S);
+
+    abs_timeout.tv_sec += sec;
+    abs_timeout.tv_nsec += nsec;
+
+    while (abs_timeout.tv_nsec > TV_NSEC_MAX) {
+      abs_timeout.tv_sec += 1;
+      abs_timeout.tv_nsec -= NS_PER_S;
+    }
+  }
 
   errcheck(sem_timedwait(psem->sem, &abs_timeout), err);
+  return OK;
 }
 
 int
 psem_getvalue(psem_t *psem, int *sval, error_t **err)
 {
   errcheck(sem_getvalue(psem->sem, sval), err);
+  return OK;
 }
+

data/lib/process_shared/abstract_semaphore.rb

@@ -8,23 +8,43 @@ module ProcessShared
     include ProcessShared::PSem
     public
 
-    # Generate a name for a semaphore.
+    # Generate a name for a semaphore.  If +name+ is given, it is used
+    # as the name (and so a semaphore could be shared by arbitrary
+    # processes not forked from one another).  Otherwise, a name is
+    # generated containing +middle+ and the process id.
+    #
+    # @param [String] middle arbitrary string used in the middle
+    # @param [String] name if given, used as the name
+    # @return [String] name, or the generated name
     def self.gen_name(middle, name = nil)
       if name
         name
       else
         @count ||= 0
         @count += 1
-        "ps-#{middle}-#{Process.pid}-#{@count}"
+        "ps-#{middle}-#{::Process.pid}-#{@count}"
       end
     end
 
+    # Make a Proc suitable for use as a finalizer that will call
+    # +psem_unlink+ on +name+ and ignore system errors.
+    #
+    # @return [Proc] a finalizer
     def self.make_finalizer(name)
       proc { ProcessShared::PSem.psem_unlink(name, nil) }
     end
 
     # private_class_method :new
 
+    def synchronize
+      wait
+      begin
+        yield
+      ensure
+        post
+      end
+    end
+
     protected
 
     attr_reader :sem, :err

data/lib/process_shared/binary_semaphore.rb

@@ -0,0 +1,48 @@
+require 'process_shared/psem'
+require 'process_shared/semaphore'
+require 'process_shared/process_error'
+
+module ProcessShared
+  # BinarySemaphore is identical to Semaphore except that its value is
+  # not permitted to rise above one (it may be either zero or one).
+  # When the value is at the maximum, calls to #post will raise an
+  # exception.
+  #
+  # This is identical to a Semaphore but with extra error checking.
+  class BinarySemaphore < Semaphore
+    # Create a new semaphore with initial value +value+.  After
+    # {Kernel#fork}, the semaphore will be shared across two (or more)
+    # processes. The semaphore must be closed with {#close} in each
+    # process that no longer needs the semaphore.
+    #
+    # (An object finalizer is registered that will close the semaphore
+    # to avoid memory leaks, but this should be considered a last
+    # resort).
+    #
+    # @param [Integer] value the initial semaphore value
+    # @param [String] name not currently supported
+    def initialize(value = 1, name = nil)
+      raise ArgumentErrror 'value must be 0 or 1' if (value < 0 or value > 1)
+      super(value, name)
+    end
+
+    # Increment from zero to one.
+    #
+    # First, attempt to decrement.  If this fails with EAGAIN, the
+    # semaphore was at zero, so continue with the post. If this
+    # succeeds, the semaphore was not at zero, so increment back to
+    # one and raise {ProcesError} (multiple workers may have acquired
+    # the semaphore at this point).
+    def post
+      begin
+        try_wait
+        # oops, value was not zero...
+        psem_post(sem, err)
+        raise ProcessError, 'post would raise value over bound'
+      rescue Errno::EAGAIN
+        # ok, value was zero
+        psem_post(sem, err)
+      end
+    end
+  end
+end

data/lib/process_shared/bounded_semaphore.rb

@@ -8,9 +8,9 @@ module ProcessShared
   class BoundedSemaphore < Semaphore
     # With no associated block, open is a synonym for
     # Semaphore.new. If the optional code block is given, it will be
-    # passed `sem` as an argument, and the Semaphore object will
+    # passed +sem+ as an argument, and the Semaphore object will
     # automatically be closed when the block terminates. In this
-    # instance, Semaphore.open returns the value of the block.
+    # instance, BoundedSemaphore.open returns the value of the block.
     #
     # @param [Integer] value the initial semaphore value
     # @param [String] name not currently supported
@@ -18,9 +18,9 @@ module ProcessShared
       new(maxvalue, value, name).with_self(&block)
     end
 
-    # Create a new semaphore with initial value `value`.  After
-    # Kernel#fork, the semaphore will be shared across two (or more)
-    # processes. The semaphore must be closed with #close in each
+    # Create a new semaphore with initial value +value+.  After
+    # {Kernel#fork}, the semaphore will be shared across two (or more)
+    # processes. The semaphore must be closed with {#close} in each
     # process that no longer needs the semaphore.
     #
     # (An object finalizer is registered that will close the semaphore

data/lib/process_shared/condition_variable.rb

@@ -1,14 +1,18 @@
 require 'process_shared/semaphore'
 
 module ProcessShared
-  # TODO: implement this
   class ConditionVariable
     def initialize
-      @sem = Semaphore.new
+      @internal = Semaphore.new(1)
+      @waiting = SharedMemory.new(:int)
+      @waiting.write_int(0)
+      @sem = Semaphore.new(0)
     end
 
     def broadcast
-      @sem.post
+      @internal.synchronize do
+        @waiting.read_int.times { @sem.post }
+      end
     end
 
     def signal
@@ -18,10 +22,32 @@ module ProcessShared
     def wait(mutex, timeout = nil)
       mutex.unlock
       begin
-        @sem.wait
+        inc_waiting
+        if timeout
+          begin
+            @sem.try_wait(timeout)
+          rescue Errno::EAGAIN, Errno::ETIMEDOUT
+            # success!
+          end
+        else
+          @sem.wait
+        end
+        dec_waiting
       ensure
         mutex.lock
       end
     end
+
+    private
+
+    def inc_waiting(val = 1)
+      @internal.synchronize do
+        @waiting.write_int(@waiting.read_int + val)
+      end
+    end
+
+    def dec_waiting
+      inc_waiting(-1)
+    end
   end
 end

data/lib/process_shared/mutex.rb

@@ -1,24 +1,24 @@
-require 'process_shared/bounded_semaphore'
+require 'process_shared/semaphore'
 require 'process_shared/with_self'
 require 'process_shared/shared_memory'
 require 'process_shared/process_error'
 
 module ProcessShared
-  # This Mutex class is implemented as a BoundedSemaphore with a
-  # maximum value of 1.  Additionally, the locking process is tracked,
-  # and ProcessError is raised if either #unlock is called by a
-  # process different from the locking process, or if #lock is called
-  # while the process already holds the lock (i.e. the mutex is not
-  # re-entrant).  This tracking is not without performance cost, of
-  # course (current implementation uses an additional BoundedSemaphore
-  # and SharedMemory segment).
+  # This Mutex class is implemented as a Semaphore with a second
+  # internal Semaphore used to track the locking process is tracked.
+  # {ProcessError} is raised if either {#unlock} is called by a
+  # process different from the locking process, or if {#lock} is
+  # called while the process already holds the lock (i.e. the mutex is
+  # not re-entrant).  This tracking is not without performance cost,
+  # of course (current implementation uses the additional {Semaphore}
+  # and {SharedMemory} segment).
   #
-  # The API is intended to be identical to the ::Mutex in the core
+  # The API is intended to be identical to the {::Mutex} in the core
   # Ruby library.
   #
   # TODO: the core Ruby api has no #close method, but this Mutex must
-  # release its BoundedSemaphore and SharedMemory resources.  For now,
-  # we rely on the object finalizers of those objects...
+  # release its {Semaphore} and {SharedMemory} resources.  For now,
+  # rely on the object finalizers of those objects...
   class Mutex
     # include WithSelf
 
@@ -27,10 +27,10 @@ module ProcessShared
     # end
 
     def initialize
-      @internal_sem = BoundedSemaphore.new(1)
+      @internal_sem = Semaphore.new
       @locked_by = SharedMemory.new(:int)
 
-      @sem = BoundedSemaphore.new(1)
+      @sem = Semaphore.new
     end
 
     # @return [Mutex]
@@ -68,7 +68,7 @@ module ProcessShared
         if @locked_by.get_int(0) > 0
           false                 # was locked
         else
-          @sem.wait
+          @sem.wait             # should return immediately
           self.locked_by = ::Process.pid
           true
         end

data/lib/process_shared/posix_call.rb

@@ -2,9 +2,9 @@
 
 module ProcessShared
   module PosixCall
-    # Replace methods in `syms` with error checking wrappers that
-    # invoke the original method and raise a SystemCallError with the
-    # current errno if the return value is an error.
+    # Replace methods in +syms+ with error checking wrappers that
+    # invoke the original method and raise a {SystemCallError} with
+    # the current errno if the return value is an error.
     #
     # Errors are detected if the block returns true when called with
     # the original method's return value.

data/lib/process_shared/psem.rb

@@ -23,6 +23,9 @@ module ProcessShared
       # Replace methods in `syms` with error checking wrappers that
       # invoke the original psem method and raise an appropriate
       # error.
+      #
+      # The last argument is assumed to be a pointer to a pointer
+      # where either a psem error or NULL will be stored.
       def psem_error_check(*syms)
         syms.each do |sym|
           method = self.method(sym)
@@ -32,7 +35,8 @@ module ProcessShared
               errp = args[-1]
               unless errp.nil?
                 begin
-                  err = Error.new(errp.get_pointer(0))
+                  err = Error.new(errp.read_pointer)
+                  errp.write_pointer(nil)
                   if err[:source] == PSem.e_source_system
                     raise SystemCallError.new("error in #{sym}", err[:errno])
                   else
@@ -86,7 +90,7 @@ module ProcessShared
     attach_function :psem_post, [:pointer, :pointer], :int
     attach_function :psem_wait, [:pointer, :pointer], :int
     attach_function :psem_trywait, [:pointer, :pointer], :int
-    attach_function :psem_timedwait, [:pointer, :pointer, :pointer], :int
+    attach_function :psem_timedwait, [:pointer, :float, :pointer], :int
     attach_function :psem_getvalue, [:pointer, :pointer, :pointer], :int
 
     psem_error_check(:psem_open, :psem_close, :psem_unlink, :psem_post,
@@ -100,7 +104,7 @@ module ProcessShared
     attach_function :bsem_post, [:pointer, :pointer], :int
     attach_function :bsem_wait, [:pointer, :pointer], :int
     attach_function :bsem_trywait, [:pointer, :pointer], :int
-    attach_function :bsem_timedwait, [:pointer, :pointer, :pointer], :int
+    attach_function :bsem_timedwait, [:pointer, :float, :pointer], :int
     attach_function :bsem_getvalue, [:pointer, :pointer, :pointer], :int
 
     psem_error_check(:bsem_open, :bsem_close, :bsem_unlink, :bsem_post,

data/lib/process_shared/semaphore.rb

@@ -5,7 +5,7 @@ module ProcessShared
   class Semaphore < AbstractSemaphore
     # With no associated block, open is a synonym for
     # Semaphore.new. If the optional code block is given, it will be
-    # passed `sem` as an argument, and the Semaphore object will
+    # passed +sem+ as an argument, and the Semaphore object will
     # automatically be closed when the block terminates. In this
     # instance, Semaphore.open returns the value of the block.
     #
@@ -15,7 +15,7 @@ module ProcessShared
       new(value, name).with_self(&block)
     end
 
-    # Create a new semaphore with initial value `value`.  After
+    # Create a new semaphore with initial value +value+.  After
     # Kernel#fork, the semaphore will be shared across two (or more)
     # processes. The semaphore must be closed with #close in each
     # process that no longer needs the semaphore.
@@ -33,18 +33,36 @@ module ProcessShared
     end
 
     # Decrement the value of the semaphore.  If the value is zero,
-    # wait until another process increments via #post.
+    # wait until another process increments via {#post}.
     def wait
       psem_wait(sem, err)
     end
 
+    # Decrement the value of the semaphore if it can be done
+    # immediately (i.e. if it was non-zero).  Otherwise, wait up to
+    # +timeout+ seconds until another process increments via {#post}.
+    #
+    # @param timeout [Numeric] the maximum seconds to wait, or nil to not wait
+    #
+    # @return If +timeout+ is nil and the semaphore cannot be
+    # decremented immediately, raise Errno::EAGAIN.  If +timeout+
+    # passed before the semaphore could be decremented, raise
+    # Errno::ETIMEDOUT.
+    def try_wait(timeout = nil)
+      if timeout
+        psem_timedwait(sem, timeout, err)
+      else
+        psem_trywait(sem, err)
+      end
+    end
+
     # Increment the value of the semaphore.  If other processes are
     # waiting on this semaphore, one will be woken.
     def post
       psem_post(sem, err)
     end
 
-    # Get the current value of the semaphore. Raises Errno::NOTSUP on
+    # Get the current value of the semaphore. Raises {Errno::NOTSUP} on
     # platforms that don't support this (e.g. Mac OS X).
     #
     # @return [Integer] the current value of the semaphore.
@@ -55,7 +73,7 @@ module ProcessShared
     end
 
     # Release the resources associated with this semaphore.  Calls to
-    # other methods are undefined after #close has been called.
+    # other methods are undefined after {#close} has been called.
     #
     # Close must be called when the semaphore is no longer needed.  An
     # object finalizer will close the semaphore as a last resort.

data/lib/process_shared/shared_array.rb

@@ -0,0 +1,69 @@
+module ProcessShared
+  class SharedArray < SharedMemory
+    include Enumerable
+
+    # A fixed-size array in shared memory.  Processes forked from this
+    # one will be able to read and write shared data to the array.
+    # Access should be synchronized using a {Mutex}, {Semaphore}, or
+    # other means.
+    #
+    # Note that {Enumerable} methods such as {#map}, {#sort},
+    # etc. return new {Array} objects rather than modifying the shared
+    # array.
+    #
+    # @param [Symbol] type_or_count the data type as a symbol
+    # understood by FFI (e.g. :int, :double)
+    #
+    # @param [Integer] count number of array elements
+    def initialize(type_or_count = 1, count = 1)
+      super(type_or_count, count)
+
+      # See https://github.com/ffi/ffi/issues/118
+      ffi_type = FFI.find_type(self.type)
+
+      name = if ffi_type.inspect =~ /FFI::Type::Builtin:(\w+)*/
+               # name will be something like int32
+               $1.downcase
+             end
+
+      unless name
+        raise ArgumentError, "could not find FFI::Type for #{self.type}"
+      end
+      
+      getter = "get_#{name}"
+      setter = "put_#{name}"
+
+      # singleton class
+      sclass = class << self; self; end
+
+      unless sclass.method_defined?(getter)
+        raise ArgumentError, "no element getter for #{self.type} (#{getter})"
+      end
+
+      unless sclass.method_defined?(setter)
+        raise ArgumentError, "no element setter for #{self.type} (#{setter})"
+      end
+
+      sclass.send(:alias_method, :get_type, getter)
+      sclass.send(:alias_method, :put_type, setter)
+    end
+
+    def each
+      # NOTE: using @count because Enumerable defines its own count
+      # method...
+      @count.times { |i| yield self[i] }
+    end
+
+    def each_with_index
+      @count.times { |i| yield self[i], i }
+    end
+
+    def [](i)
+      get_type(i * self.type_size)
+    end
+
+    def []=(i, val)
+      put_type(i * self.type_size, val)
+    end
+  end
+end

data/lib/process_shared/shared_memory.rb

@@ -1,25 +1,37 @@
 require 'process_shared/rt'
 require 'process_shared/libc'
 require 'process_shared/with_self'
+require 'process_shared/shared_memory_io'
 
 module ProcessShared
-  # Memory block shared across processes. TODO: finalizer that closes...
+  # Memory block shared across processes.
   class SharedMemory < FFI::Pointer
     include WithSelf
 
-    attr_reader :size, :fd
+    attr_reader :size, :type, :type_size, :count, :fd
 
     def self.open(size, &block)
       new(size).with_self(&block)
     end
 
-    def initialize(size)
-      @size = case size
-              when Symbol
-                FFI.type_size(size)
-              else
-                size
-              end
+    def self.make_finalizer(addr, size, fd)
+      proc do
+        pointer = FFI::Pointer.new(addr)
+        LibC.munmap(pointer, size)
+        LibC.close(fd)
+      end
+    end
+
+    def initialize(type_or_count = 1, count = 1)
+      @type, @count = case type_or_count
+                      when Symbol
+                        [type_or_count, count]
+                      else
+                        [:uchar, type_or_count]
+                      end
+
+      @type_size = FFI.type_size(@type)
+      @size = @type_size * @count
 
       name = "/ps-shm#{rand(10000)}"
       @fd = RT.shm_open(name,
@@ -33,13 +45,69 @@ module ProcessShared
                            LibC::PROT_READ | LibC::PROT_WRITE,
                            LibC::MAP_SHARED,
                            @fd,
-                           0)
+                           0).
+        slice(0, size) # slice to get FFI::Pointer that knows its size
+                       # (and thus does bounds checking)
+
+      @finalize = self.class.make_finalizer(@pointer.address, @size, @fd)
+      ObjectSpace.define_finalizer(self, @finalize)
+
       super(@pointer)
     end
 
+    # Write the serialization of +obj+ (using Marshal.dump) to this
+    # shared memory object at +offset+ (in bytes).
+    #
+    # Raises IndexError if there is insufficient space.
+    def put_object(offset, obj)
+      # FIXME: This is a workaround to an issue I'm seeing in
+      # 1.8.7-p352 (not tested in other 1.8's).  If I used the code
+      # below that works in 1.9, then inside SharedMemoryIO#write, the
+      # passed string object is 'terminated' (garbage collected?) and
+      # won't respond to any methods...  This way is less efficient
+      # since it involves the creation of an intermediate string, but
+      # it works in 1.8.7-p352.
+      if RUBY_VERSION =~ /^1.8/
+        str = Marshal.dump(obj)
+        return put_bytes(offset, str, 0, str.size)
+      end
+
+      io = SharedMemoryIO.new(self)
+      io.seek(offset)
+      Marshal.dump(obj, io)
+    end
+
+    # Read the serialized object at +offset+ (in bytes) using
+    # Marshal.load.
+    #
+    # @return [Object]
+    def get_object(offset)
+      io = to_shm_io
+      io.seek(offset)
+      Marshal.load(io)
+    end
+
+    # Equivalent to {#put_object(0, obj)}
+    def write_object(obj)
+      put_object(0, obj)
+    end
+
+    # Equivalent to {#read_object(0, obj)}
+    #
+    # @return [Object]
+    def read_object
+      Marshal.load(to_shm_io)
+    end
+
     def close
-      LibC.munmap(@pointer, @size)
-      LibC.close(@fd)
+      ObjectSpace.undefine_finalizer(self)
+      @finalize.call
+    end
+
+    private
+
+    def to_shm_io
+      SharedMemoryIO.new(self)
     end
   end
 end