cod 0.4.0 → 0.4.1

data/Gemfile

@@ -1,18 +1,3 @@
 source "http://rubygems.org"
 
-gem 'rake'
-gem 'beanstalk-client'
-
-group :test do
-  gem 'rspec'
-  gem 'flexmock'
-  
-  gem 'guard'
-  gem 'guard-rspec'
-  
-  gem 'rdoc'
-  gem 'sdoc'
-  
-  gem 'growl'
-  gem 'rb-fsevent'
-end
+gemspec

data/HISTORY.txt

@@ -1,3 +1,15 @@
+= 0.4.1 / 23Jan2012
+  + Cod.stdio returns a channel that is linked to this process' stdin and
+    stdout. 
+
+  + Cod.process spawns a process and allows communicating with that process
+    via stdin/stdout.
+    
+  + Cod.bidir_pipe spawns two Cod.pipe's, one for writing to the other end, 
+    one for reading from it. (see above...)
+    
+  + Small fixes, preparation for connection robustness
+
 == 0.4.0 / 29Nov2011
 
   * A complete rewrite, focusing on lean implementation and logical/useful 

data/README

@@ -1,9 +1,9 @@
+Some people, when confronted with a problem, think "I know, I'll use
+multithreading". Nothhw tpe yawrve o oblems.[1]
 
-IPC for babies and those who want to become. Another thin API layer over what
-Ruby offers for IO.pipe, TCP sockets and FIFOs: Makes sending and receiving
-of Ruby objects a breeze. 
-
-A good place to start is the documentation for the Cod module.
+Don't use threads, use processes. This is the toolkit for interprocess
+communication that makes it real simple. A good place to start is the
+documentation for the Cod module.
 
 SYNOPSIS
 
@@ -29,4 +29,6 @@ that we had before, but some things have been designed more logically.
 
 At version 0.4.0
 
-(c) 2011 Kaspar Schiess
+(c) 2011 Kaspar Schiess
+
+[1] https://twitter.com/rogerbraun/status/160813717502705664

data/Rakefile

@@ -19,7 +19,7 @@ require 'sdoc'
 
 # Generate documentation
 RDoc::Task.new do |rdoc|
-  rdoc.title    = "parslet - construction of parsers made easy"
+  rdoc.title    = "cod - IPC made really simple."
   rdoc.options << '--line-numbers'
   rdoc.options << '--fmt' << 'shtml' # explictly set shtml generator
   rdoc.template = 'direct' # lighter template used on railsapi.com

data/examples/netcat/README

@@ -0,0 +1,2 @@
+Shows that you can use netcat as a proxy between Cod.process and
+Cod.tcp_server...

data/examples/netcat/server.rb

@@ -0,0 +1,25 @@
+$:.unshift File.expand_path(File.dirname(__FILE__) + "/../../lib")
+require 'cod'
+
+fork do
+  server = Cod.tcp_server('localhost:12345')
+
+  request, channel = server.get_ext
+  case request
+    when :get_time
+      channel.put Time.now
+  else 
+    fail
+  end
+end
+
+fork do
+  sleep 1
+  process = Cod.process('nc localhost 12345')
+  channel = process.channel
+  
+  channel.put :get_time
+  puts channel.get
+end
+
+Process.waitall

data/examples/protocol-buffers/README

@@ -0,0 +1,3 @@
+A demonstration of what it takes to use googles protocol buffers (protobuf) on
+the wire. You will need the ruby-protocol-buffers gem for this to work. 
+

data/examples/protocol-buffers/master_child.rb

@@ -0,0 +1,27 @@
+$:.unshift File.expand_path(File.dirname(__FILE__) + "/../../lib")
+require 'cod'
+require 'cod/protocol_buffers_serializer'
+
+require 'protocol_buffers'
+require 'protocol_buffers/compiler'
+
+
+ProtocolBuffers::Compiler.compile_and_load_string <<-EOS
+message Foo {
+  required string bar = 1;
+};
+EOS
+
+pipe = Cod.pipe(Cod::ProtocolBuffersSerializer.new)
+
+child_pid = fork do
+  pipe.put Foo.new(:bar => 'bar')
+  pipe.put Foo.new(:bar => 'baz')
+end
+
+begin
+  p pipe.get
+  p pipe.get
+ensure
+  Process.wait(child_pid)
+end

data/lib/cod.rb

@@ -26,11 +26,20 @@ module Cod
   # Creates a pipe connection that is visible to this process and its
   # children. (see Cod::Pipe)
   #
-  def pipe
-    Cod::Pipe.new
+  def pipe(serializer=nil, pipe_pair=nil)
+    Cod::Pipe.new(serializer)
   end
   module_function :pipe
   
+  # Creates two channels based on Cod.pipe (unidirectional IO.pipe) and links
+  # things up so that you communication is bidirectional. Writes go to 
+  # #out and reads come from #in.
+  #
+  def bidir_pipe(serializer=nil, pipe_pair=nil)
+    Cod::BidirPipe.new(serializer, pipe_pair)
+  end
+  module_function :bidir_pipe
+  
   # Creates a tcp connection to the destination and returns a channel for it.
   # (see Cod::TcpClient)
   #
@@ -57,6 +66,25 @@ module Cod
   end
   module_function :beanstalk
 
+  # Runs a command via Process.spawn, then links a channel to the commands
+  # stdout and stdin. Returns the commands pid and the channel. 
+  #
+  # Example: 
+  #   pid, channel = Cod.process('cat')
+  def process(command, serializer=nil)
+    Cod::Process.new(command, serializer)
+  end
+  module_function :process
+  
+  # Links a process' stdin and stdout up with a pipe. This means that the
+  # pipes #put method will print to stdout, and the #get method will read from 
+  # stdin.
+  #
+  def stdio(serializer=nil)
+    Cod::Pipe.new(serializer, [$stdin, $stdout])
+  end
+  module_function :stdio
+  
   # Indicates that the given channel is write only. This gets raised on 
   # operations like #put.
   #
@@ -79,11 +107,16 @@ end
 require 'cod/select_group'
 require 'cod/select'
 
+require 'cod/iopair'
+
 require 'cod/channel'
 
 require 'cod/simple_serializer'
 
 require 'cod/pipe'
+require 'cod/bidir_pipe'
+
+require 'cod/process'
 
 require 'cod/tcp_client'
 require 'cod/tcp_server'

data/lib/cod/beanstalk/channel.rb

@@ -11,13 +11,27 @@ module Cod::Beanstalk
   class Channel < Cod::Channel
     JOB_PRIORITY = 0
     
+    # Which tube this channel is connected to
+    attr_reader :tube_name
+    # Beanstalkd server this channel is connected to
+    attr_reader :server_url
+    
     def initialize(tube_name, server_url)
+      super()
       @tube_name, @server_url = tube_name, server_url
     
       @body_serializer = Cod::SimpleSerializer.new
       @transport = connection(server_url, tube_name)
     end
     
+    # Allow #dup on beanstalkd channels, resulting in a _new_ connection to
+    # the beanstalkd server. 
+    #
+    def initialize_copy(other)
+      super(other)
+      initialize(other.tube_name, other.server_url)
+    end
+    
     def put(msg)
       pri   = JOB_PRIORITY
       delay = 0

data/lib/cod/bidir_pipe.rb

@@ -0,0 +1,32 @@
+module Cod
+  class BidirPipe < Channel
+    # The Cod::Pipe instance we're currently #put'ting to. 
+    attr_reader :w
+    # The Cod::Pipe instance we're currently #get'ting from. 
+    attr_reader :r
+    
+    def initialize(serializer=nil, pipe_pair=nil)
+      @serializer = serializer || SimpleSerializer.new
+      @r, @w = pipe_pair || [Cod.pipe(@serializer), Cod.pipe(@serializer)]
+    end
+    
+    def put(msg)
+      w.put(msg)
+    end
+    
+    def get
+      r.get
+    end
+    
+    def close
+      r.close
+      w.close
+    end
+    
+    # Swaps the end of this pipe around. 
+    #
+    def swap!
+      @r, @w = w, r
+    end
+  end
+end

data/lib/cod/iopair.rb

@@ -0,0 +1,40 @@
+module Cod
+  IOPair = Struct.new(:r, :w) do
+    def initialize(r=nil, w=nil)
+      if r && w
+        super(r, w)
+      else
+        super(*IO.pipe)
+      end
+    end
+    
+    # Performs a deep copy of the structure. 
+    def initialize_copy(other)
+      super
+      self.r = other.r.dup if other.r
+      self.w = other.w.dup if other.w
+    end
+    def write(buf)
+      close_r
+      raise Cod::ReadOnlyChannel unless w
+      w.write(buf)
+    end
+    def read(serializer)
+      close_w
+      raise Cod::WriteOnlyChannel unless r
+      serializer.de(r)
+    end
+    def close
+      close_r
+      close_w
+    end
+    def close_r
+      r.close if r
+      self.r = nil
+    end
+    def close_w
+      w.close if w
+      self.w = nil
+    end
+  end
+end

data/lib/cod/pipe.rb

@@ -12,36 +12,10 @@ module Cod
   # create the channel before forking the child, since master and child will
   # share all objects that were available before the fork. 
   #
-  class Pipe
+  class Pipe < Channel
     attr_reader :pipe
     attr_reader :serializer
     
-    IOPair = Struct.new(:r, :w) do
-      # Performs a deep copy of the structure. 
-      def initialize_copy(other)
-        super
-        self.r = other.r.dup if other.r
-        self.w = other.w.dup if other.w
-      end
-      def write(buf)
-        close_r
-        raise Cod::ReadOnlyChannel unless w
-        w.write(buf)
-      end
-      def close
-        close_r
-        close_w
-      end
-      def close_r
-        r.close if r
-        self.r = nil
-      end
-      def close_w
-        w.close if w
-        self.w = nil
-      end
-    end
-
     # A few methods that a pipe split must answer to. The split itself is 
     # basically an array instance; these methods add some calling safety and
     # convenience. 
@@ -51,10 +25,10 @@ module Cod
       def write; last; end
     end
 
-    def initialize(serializer=nil)
-      super
+    def initialize(serializer=nil, pipe_pair=nil)
+      super()
       @serializer = serializer || SimpleSerializer.new
-      @pipe = IOPair.new(*IO.pipe)
+      @pipe = IOPair.new(*pipe_pair)
     end
     
     # Creates a copy of this pipe channel. This performs a shallow #dup except
@@ -148,13 +122,13 @@ module Cod
       not result.nil?
     end
     def to_read_fds
-      pipe.r
+      r
     end
     
     # Returns true if you can read from this pipe. 
     #
     def can_read?
-      not pipe.r.nil?
+      not r.nil?
     end
     
     # Returns true if you can write to this pipe. 
@@ -163,6 +137,20 @@ module Cod
       not pipe.w.nil?
     end
 
+    # ------------------------------------------------------- internal helpers
+
+    # Returns the read end of the pipe
+    #
+    def r
+      pipe.r
+    end
+    
+    # Returns the write end of the pipe
+    #
+    def w
+      pipe.w
+    end
+
     # --------------------------------------------------------- service/client
     
     def service
@@ -182,7 +170,7 @@ module Cod
   private
     def deserialize_one
       # Now deserialize one message from the buffer in io
-      serializer.de(pipe.r)
+      pipe.read(serializer)
     end
   end
 end

data/lib/cod/process.rb

@@ -0,0 +1,34 @@
+module Cod
+  class Process
+    attr_reader :pid 
+    
+    def initialize(command, serializer=nil)
+      @serializer = serializer || SimpleSerializer.new
+
+      run(command)
+    end
+    
+    def run(command)
+      @pipe = Cod.bidir_pipe(@serializer)
+      
+      @pid = ::Process.spawn(command, 
+        :in => @pipe.w.r, 
+        :out => @pipe.r.w)
+    end
+    
+    def channel
+      @pipe
+    end
+    
+    def kill
+      ::Process.kill :TERM, @pid
+    end
+    def terminate
+      @pipe.w.close
+    end
+    
+    def wait
+      ::Process.wait(@pid)
+    end
+  end
+end

data/lib/cod/protocol_buffers_serializer.rb

@@ -0,0 +1,59 @@
+# Install the ruby-protocol-buffers gem for this to work. 
+require 'protocol_buffers'
+
+module Cod
+  # Serializes a protocol buffer (googles protobufs) message to the wire and
+  # reads it back. Protobufs are not self-delimited, this is why we store the
+  # messages in the following on-wire format: 
+  #
+  #   LEN(class) BYTES(class) LEN(message) BYTES(message)
+  #
+  # where LEN is a varint representation of the length of the contained item
+  # and BYTES are the binary bytes of said item. 
+  #
+  # This is not the most space efficient manner of representing things on the
+  # wire. It also assumes that you have defined the message classes on both
+  # sides (client and server). 
+  #
+  # For applications where this is a problem, you can always use this
+  # implementation as a guide for your own implementation. For example,
+  # message polymorphism could be coded as a single byte on the wire, allowing
+  # for one of 255 messages to be sent each time. For really efficient
+  # transfers, you could even send a fixed amount of bytes and one message,
+  # getting the most out of protobufs. 
+  #
+  # Please see examples/protocol-buffers/master_child.rb for information
+  # on how to use this. 
+  #
+  class ProtocolBuffersSerializer
+    Varint = ProtocolBuffers::Varint
+    
+    def en(obj)
+      sio = ProtocolBuffers.bin_sio
+      
+      # Assuming that obj is a protocol buffers message object, this should 
+      # work: 
+      klass_name = obj.class.name
+      buffer = obj.to_s
+
+      Varint.encode(sio, klass_name.size)
+      sio.write(klass_name)
+      
+      Varint.encode(sio, buffer.size)
+      sio.write(buffer)
+      
+      sio.string
+    end
+    
+    def de(io)
+      klass_size = Varint.decode(io)
+      klass_name = io.read(klass_size)
+      
+      klass = self.class.const_get(klass_name)
+      
+      msg_size = Varint.decode(io)
+      limited_io = LimitedIO.new(io, msg_size)
+      klass.parse(limited_io)
+    end
+  end
+end

data/lib/cod/tcp_server.rb

@@ -1,4 +1,33 @@
-module Cod
+require 'socket'
+
+module Cod       
+  # A tcp server channel. Messages are read from any of the connected sockets
+  # in a round robin fashion. 
+  #
+  # Synopsis: 
+  #   server = Cod.tcp_server('localhost:12345') 
+  #   server.get  # 'a message'
+  #   msg, chan = server.get_ext
+  #
+  # There is no implementation of #put that would broadcast back to all
+  # connected sockets, this is up to you to implement. Instead, you can use
+  # one of two ways to obtain a channel for talking back to a specific client:
+  # 
+  # Using #get_ext:
+  #   msg, chan = server.get_ext 
+  #
+  # chan is a two way connected channel to the specific client that has opened 
+  # its communication with msg. 
+  #
+  # Using plain #get: 
+  #   # on the client: 
+  #   client.put [client, :msg]
+  #   # on the server
+  #   chan, msg = server.get
+  #
+  # This means that you can transmit the client channel through the connection 
+  # as part of the message you send. 
+  # 
   class TcpServer
     def initialize(bind_to)
       @socket = TCPServer.new(*bind_to.split(':'))

data/lib/tcp_proxy.rb

@@ -0,0 +1,132 @@
+class TCPProxy
+  attr_reader :connections
+  
+  def initialize(host, from_port, to_port)
+    @ins  = TCPServer.new(host, from_port)
+
+    @host = host
+    @from_port, @to_port = from_port, to_port
+
+    # Active connections and mutex to protect access to it.
+    @connections = []
+    @connections_m = Mutex.new
+    
+    # Are we currently accepting new connections?
+    @accept_new = true
+    
+    @thread = Thread.start(&method(:thread_main))
+  end
+  
+  def close
+    @shutdown = true
+    
+    @thread.join
+    @thread = nil
+    
+    # Since the thread is stopped now, we can be sure no new connections are
+    # accepted. This is why we access the collection without locking.
+    @connections.each do |connection|
+      connection.close
+    end
+    @ins.close
+  end
+  
+  def block
+    @accept_new = false
+  end
+  def allow
+    @accept_new = true
+  end
+  
+  def drop_all
+    # Copy the connections and then empty the collection
+    connections = @connections_m.synchronize {
+      @connections.tap { 
+        @connections = [] } }
+    
+    connections.each do |conn|
+      conn.close
+    end
+  end
+  
+  # Inside the background thread ----------------------------------------
+  
+  def thread_main
+    loop do
+      accept_connections if @accept_new
+      
+      forward_data
+      
+      break if @shutdown
+    end
+  rescue Exception => ex
+    p [:uncaught, ex]
+    ex.backtrace.each do |line|
+      puts line
+    end
+    raise
+  end
+
+  class Connection
+    def initialize(in_sock, out_sock)
+      @m = Mutex.new
+      @in_sock, @out_sock = in_sock, out_sock
+    end
+    
+    def close
+      @m.synchronize {
+        @in_sock.close; @in_sock = nil
+        @out_sock.close; @out_sock = nil }
+    end
+
+    def pump_synchronized(n=10)
+      @m.synchronize {
+        return unless @in_sock && @out_sock
+        pump(n) }
+    end
+    
+    def pump(n)
+      while n>0
+        available_sockets = [@in_sock, @out_sock]
+        ready_sockets, (*) = IO.select(available_sockets, nil, nil, 0)
+
+        break unless ready_sockets && !ready_sockets.empty?
+        
+        ready_sockets.each do |socket|
+          buf = socket.read_nonblock(16*1024)
+          
+          if socket == @in_sock
+            puts "--> #{buf.size}"
+            @out_sock.write(buf)
+          else
+            puts "<-- #{buf.size}"
+            @in_sock.write(buf)
+          end
+        end
+
+        n -= 1
+      end
+    rescue Errno::EAGAIN
+      # Read would block, attempt later
+    end
+  end
+
+  def accept_connections
+    loop do
+      in_sock = @ins.accept_nonblock
+      out_sock = TCPSocket.new(@host, @to_port)
+      
+      @connections_m.synchronize {
+        @connections << Connection.new(in_sock, out_sock) }
+    end
+  rescue Errno::EAGAIN
+    # No more connections pending, stop accepting new connections
+  end
+  
+  def forward_data
+    connections = @connections_m.synchronize { @connections.dup }
+    connections.each do |conn|
+      conn.pump_synchronized
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cod
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-11-29 00:00:00.000000000Z
+date: 2012-01-23 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70299431388260 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70299431388260
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70297088779060 !ruby/object:Gem::Requirement
+  requirement: &70299431387140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +32,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70297088779060
+  version_requirements: *70299431387140
 - !ruby/object:Gem::Dependency
   name: flexmock
-  requirement: &70297088777820 !ruby/object:Gem::Requirement
+  requirement: &70299431386460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +43,21 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70297088777820
+  version_requirements: *70299431386460
 - !ruby/object:Gem::Dependency
   name: sdoc
-  requirement: &70297088776720 !ruby/object:Gem::Requirement
+  requirement: &70299431385340 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70299431385340
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: &70299431384600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,7 +65,18 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70297088776720
+  version_requirements: *70299431384600
+- !ruby/object:Gem::Dependency
+  name: growl
+  requirement: &70299431383600 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70299431383600
 description: 
 email: kaspar.schiess@absurd.li
 executables: []
@@ -60,8 +93,12 @@ files:
 - lib/cod/beanstalk/serializer.rb
 - lib/cod/beanstalk/service.rb
 - lib/cod/beanstalk.rb
+- lib/cod/bidir_pipe.rb
 - lib/cod/channel.rb
+- lib/cod/iopair.rb
 - lib/cod/pipe.rb
+- lib/cod/process.rb
+- lib/cod/protocol_buffers_serializer.rb
 - lib/cod/select.rb
 - lib/cod/select_group.rb
 - lib/cod/service.rb
@@ -70,12 +107,17 @@ files:
 - lib/cod/tcp_server.rb
 - lib/cod/work_queue.rb
 - lib/cod.rb
+- lib/tcp_proxy.rb
 - examples/example_scaffold.rb
 - examples/master_child.rb
+- examples/netcat/README
+- examples/netcat/server.rb
 - examples/ping_pong/ping.rb
 - examples/ping_pong/pong.rb
 - examples/presence/client.rb
 - examples/presence/server.rb
+- examples/protocol-buffers/master_child.rb
+- examples/protocol-buffers/README
 - examples/queue/client.rb
 - examples/queue/queue.rb
 - examples/queue/README
@@ -98,13 +140,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3574736004851783821
+      hash: 1455579630017282096
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 1455579630017282096
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.10