einhorn 0.4.9 → 0.5.0

data/History.txt

@@ -0,0 +1,4 @@
+=== 0.5.0 2013-07-11
+
+* 1 major enhancement:
+  * Improve Einhorn protocol to allow multiplexing of commands on a single socket

data/bin/einhornsh

@@ -10,11 +10,33 @@ require 'einhorn'
 
 module Einhorn
   class EinhornSH
+
     def initialize(path_to_socket)
       @path_to_socket = path_to_socket
+      @request_id = 0
       reconnect
     end
 
+    def send_command(hash)
+      begin
+        @client.send_command(hash)
+        while response = @client.receive_message
+          if response.kind_of?(Hash)
+            yield response['message']
+            return unless response['wait']
+          else
+            puts "Invalid response type #{response.class}: #{response.inspect}"
+          end
+        end
+      rescue Errno::EPIPE => e
+        emit("einhornsh: Error communicating with Einhorn: #{e} (#{e.class})")
+        emit("einhornsh: Attempting to reconnect...")
+        reconnect
+
+        retry
+      end
+    end
+
     def run
       emit("Enter 'help' if you're not sure what to do.")
       emit
@@ -26,25 +48,16 @@ module Einhorn
           emit("Goodbye!")
           return
         end
-
-        begin
-          response = @client.command({'command' => command, 'args' => args})
-        rescue Errno::EPIPE => e
-          emit("einhornsh: Error communicating with Einhorn: #{e} (#{e.class})")
-          emit("einhornsh: Attempting to reconnect...")
-          reconnect
-
-          retry
-        end
-
-        if response.kind_of?(Hash)
-          puts response['message']
-        else
-          puts "Invalid response type #{response.class}: #{response.inspect}"
+        send_command({'id' => request_id, 'command' => command, 'args' => args}) do |message|
+          puts message
         end
       end
     end
 
+    def request_id
+      @request_id += 1
+    end
+
     def parse_command(line)
       command, *args = Shellwords.shellsplit(line)
       [command, args]
@@ -69,8 +82,9 @@ EOF
     end
 
     def ehlo
-      response = @client.command('command' => 'ehlo', 'user' => ENV['USER'])
-      emit(response['message'])
+      send_command({'command' => "ehlo", 'user' => ENV['USER']}) do |message|
+        emit(message)
+      end
     end
 
     def self.emit(message=nil, force=false)

data/lib/einhorn/client.rb

@@ -7,6 +7,9 @@ module Einhorn
     # Keep this in this file so client can be loaded entirely
     # standalone by user code.
     module Transport
+
+      ParseError = defined?(Psych::SyntaxError) ? Psych::SyntaxError : ArgumentError
+
       def self.send_message(socket, message)
         line = serialize_message(message)
         socket.write(line)
@@ -29,8 +32,6 @@ module Einhorn
       end
     end
 
-    @@responseless_commands = Set.new(['worker:ack'])
-
     def self.for_path(path_to_socket)
       socket = UNIXSocket.open(path_to_socket)
       self.new(socket)
@@ -45,13 +46,12 @@ module Einhorn
       @socket = socket
     end
 
-    def command(command_hash)
+    def send_command(command_hash)
       Transport.send_message(@socket, command_hash)
-      Transport.receive_message(@socket) if expect_response?(command_hash)
     end
 
-    def expect_response?(command_hash)
-      !@@responseless_commands.include?(command_hash['command'])
+    def receive_message
+      Transport.receive_message(@socket)
     end
 
     def close

data/lib/einhorn/command/interface.rb

@@ -202,38 +202,50 @@ module Einhorn::Command
     end
 
     def self.process_command(conn, command)
-      response = generate_response(conn, command)
-      if !response.nil?
-        send_message(conn, response)
+      begin
+        request = Einhorn::Client::Transport.deserialize_message(command)
+      rescue Einhorn::Client::Transport::ParseError
+      end
+      unless request.kind_of?(Hash)
+        send_message(conn, "Could not parse command")
+        return
+      end
+
+      message = generate_message(conn, request)
+      if !message.nil?
+        send_message(conn, message, request['id'], true)
       else
         conn.log_debug("Got back nil response, so not responding to command.")
       end
     end
 
-    def self.send_message(conn, response)
-      if response.kind_of?(String)
-        response = {'message' => response}
+    def self.send_tagged_message(tag, message, last=false)
+      Einhorn::Event.connections.each do |conn|
+        if id = conn.subscription(tag)
+          self.send_message(conn, message, id, last)
+          conn.unsubscribe(tag) if last
+        end
       end
-      Einhorn::Client::Transport.send_message(conn, response)
     end
 
-    def self.generate_response(conn, command)
-      begin
-        request = Einhorn::Client::Transport.deserialize_message(command)
-      rescue ArgumentError => e
-        return {
-          'message' => "Could not parse command: #{e}"
-        }
+    def self.send_message(conn, message, request_id=nil, last=false)
+      if request_id
+        response = {'message' => message, 'request_id' => request_id }
+        response['wait'] = true unless last
+      else
+        # support old-style protocol
+        response = {'message' => message}
       end
+      Einhorn::Client::Transport.send_message(conn, response)
+    end
 
+    def self.generate_message(conn, request)
       unless command_name = request['command']
-        return {
-          'message' => 'No "command" parameter provided; not sure what you want me to do.'
-        }
+        return 'No "command" parameter provided; not sure what you want me to do.'
       end
 
       if command_spec = @@commands[command_name]
-        conn.log_debug("Received command: #{command.inspect}")
+        conn.log_debug("Received command: #{request.inspect}")
         begin
           return command_spec[:code].call(conn, request)
         rescue StandardError => e
@@ -242,7 +254,7 @@ module Einhorn::Command
           return msg
         end
       else
-        conn.log_debug("Received unrecognized command: #{command.inspect}")
+        conn.log_debug("Received unrecognized command: #{request.inspect}")
         return unrecognized_command(conn, request)
       end
     end
@@ -295,7 +307,7 @@ EOF
       YAML.dump(Einhorn::Command.dumpable_state)
     end
 
-    command 'reload', 'Reload Einhorn' do |conn, _|
+    command 'reload', 'Reload Einhorn' do |conn, request|
       # TODO: make reload actually work (command socket reopening is
       # an issue). Would also be nice if user got a confirmation that
       # the reload completed, though that's not strictly necessary.
@@ -303,7 +315,7 @@ EOF
       # In the normal case, this will do a write
       # synchronously. Otherwise, the bytes will be stuck into the
       # buffer and lost upon reload.
-      send_message(conn, 'Reloading, as commanded')
+      send_message(conn, 'Reloading, as commanded', request['id'], true)
       Einhorn::Command.reload
     end
 
@@ -323,10 +335,13 @@ EOF
       Einhorn::Command.louder
     end
 
-    command 'upgrade', 'Upgrade all Einhorn workers. This may result in Einhorn reloading its own code as well.' do |conn, _|
-      # TODO: send confirmation when this is done
-      send_message(conn, 'Upgrading, as commanded')
-      # This or may not return
+    command 'upgrade', 'Upgrade all Einhorn workers. This may result in Einhorn reloading its own code as well.' do |conn, request|
+      # send first message directly for old clients that don't support request
+      # ids or subscriptions. Everything else is sent tagged with request id
+      # for new clients.
+      send_message(conn, 'Upgrading, as commanded', request['id'])
+      conn.subscribe(:upgrade, request['id'])
+      # If the app is preloaded this doesn't return.
       Einhorn::Command.full_upgrade
       nil
     end

data/lib/einhorn/command.rb

@@ -301,10 +301,10 @@ module Einhorn
 
     def self.upgrade_workers
       if Einhorn::State.upgrading
-        Einhorn.log_info("Currently upgrading (#{Einhorn::WorkerPool.ack_count} / #{Einhorn::WorkerPool.ack_target} ACKs; bumping version and starting over)...")
+        Einhorn.log_info("Currently upgrading (#{Einhorn::WorkerPool.ack_count} / #{Einhorn::WorkerPool.ack_target} ACKs; bumping version and starting over)...", :upgrade)
       else
         Einhorn::State.upgrading = true
-        Einhorn.log_info("Starting upgrade to #{Einhorn::State.version}...")
+        Einhorn.log_info("Starting upgrade from version #{Einhorn::State.version}...", :upgrade)
       end
 
       # Reset this, since we've just upgraded to a new universe (I'm
@@ -323,7 +323,8 @@ module Einhorn
 
       if Einhorn::State.upgrading && acked >= target
         Einhorn::State.upgrading = false
-        Einhorn.log_info("Upgrade to version #{Einhorn::State.version} complete.")
+        Einhorn.log_info("Upgraded successfully to version #{Einhorn::State.version} (Einhorn #{Einhorn::VERSION}).", :upgrade)
+        Einhorn.send_tagged_message(:upgrade, "Upgrade done", true)
       end
 
       old_workers = Einhorn::WorkerPool.old_workers

data/lib/einhorn/event/connection.rb

@@ -2,6 +2,11 @@ module Einhorn::Event
   class Connection < AbstractTextDescriptor
     include Persistent
 
+    def initialize(*args)
+      @subscriptions = {}
+      super
+    end
+
     def parse_record
       split = @read_buffer.split("\n", 2)
       if split.length > 1
@@ -20,6 +25,7 @@ module Einhorn::Event
       # Don't include by default because it's not that pretty
       state[:read_buffer] = @read_buffer if @read_buffer.length > 0
       state[:write_buffer] = @write_buffer if @write_buffer.length > 0
+      state[:subscriptions] = @subscriptions
       state
     end
 
@@ -29,16 +35,36 @@ module Einhorn::Event
       conn = self.open(socket)
       conn.read_buffer = state[:read_buffer] if state[:read_buffer]
       conn.write_buffer = state[:write_buffer] if state[:write_buffer]
+      # subscriptions could be empty if upgrading from an older version of einhorn
+      state.fetch(:subscriptions, {}).each do |tag, id|
+        conn.subscribe(tag, id)
+      end
       conn
     end
 
+    def subscribe(tag, request_id)
+      if request_id
+        @subscriptions[tag] = request_id
+      end
+    end
+
+    def subscription(tag)
+      @subscriptions[tag]
+    end
+
+    def unsubscribe(tag)
+      @subscriptions.delete(tag)
+    end
+
     def register!
       log_info("client connected")
+      Einhorn::Event.register_connection(self, @socket.fileno)
       super
     end
 
     def deregister!
       log_info("client disconnected") if Einhorn::TransientState.whatami == :master
+      Einhorn::Event.deregister_connection(@socket.fileno)
       super
     end
   end

data/lib/einhorn/event/persistent.rb

@@ -11,7 +11,7 @@ module Einhorn::Event
       if klass = @@persistent[klass_name]
         klass.from_state(state)
       else
-        Einhorn.log_error("Unrecognized persistent descriptor class #{klass_name.inspect}. Ignoring. This most likely indicates that your Einhorn version has upgraded. Everything should still be working, but it may be worth a restart.")
+        Einhorn.log_error("Unrecognized persistent descriptor class #{klass_name.inspect}. Ignoring. This most likely indicates that your Einhorn version has upgraded. Everything should still be working, but it may be worth a restart.", :upgrade)
         nil
       end
     end

data/lib/einhorn/event.rb

@@ -7,6 +7,7 @@ module Einhorn
     @@signal_actions = []
     @@readable = {}
     @@writeable = {}
+    @@connections = {}
     @@timers = {}
 
     def self.cloexec!(fd)
@@ -88,6 +89,18 @@ module Einhorn
       writers
     end
 
+    def self.register_connection(connection, fd)
+      @@connections[fd] = connection
+    end
+
+    def self.deregister_connection(fd)
+      @@connections.delete(fd)
+    end
+
+    def self.connections
+      @@connections.values
+    end
+
     def self.register_timer(timer)
       @@timers[timer.expires_at] ||= Set.new
       @@timers[timer.expires_at] << timer

data/lib/einhorn/version.rb

@@ -1,3 +1,3 @@
 module Einhorn
-  VERSION = '0.4.9'
+  VERSION = '0.5.0'
 end

data/lib/einhorn/worker.rb

@@ -69,7 +69,7 @@ module Einhorn
         raise "Unrecognized socket discovery mechanism: #{discovery.inspect}. Must be one of :filesystem, :argv, or :direct"
       end
 
-      client.command({
+      client.send_command({
           'command' => 'worker:ack',
           'pid' => $$
         })

data/lib/einhorn.rb

@@ -169,11 +169,17 @@ module Einhorn
   def self.log_debug(msg)
     $stderr.puts("#{log_tag} DEBUG: #{msg}") if Einhorn::State.verbosity <= 0
   end
-  def self.log_info(msg)
+  def self.log_info(msg, tag=nil)
     $stderr.puts("#{log_tag} INFO: #{msg}") if Einhorn::State.verbosity <= 1
+    self.send_tagged_message(tag, msg) if tag
   end
-  def self.log_error(msg)
+  def self.log_error(msg, tag=nil)
     $stderr.puts("#{log_tag} ERROR: #{msg}") if Einhorn::State.verbosity <= 2
+    self.send_tagged_message(tag, "ERROR: #{msg}") if tag
+  end
+
+  def self.send_tagged_message(tag, message, last=false)
+    Einhorn::Command::Interface.send_tagged_message(tag, message, last)
   end
 
   private
@@ -221,20 +227,20 @@ module Einhorn
       begin
         # If it's not going to be requireable, then load it.
         if !path.end_with?('.rb') && File.exists?(path)
-          log_info("Loading #{path} (if this hangs, make sure your code can be properly loaded as a library)")
+          log_info("Loading #{path} (if this hangs, make sure your code can be properly loaded as a library)", :upgrade)
           load path
         else
-          log_info("Requiring #{path} (if this hangs, make sure your code can be properly loaded as a library)")
+          log_info("Requiring #{path} (if this hangs, make sure your code can be properly loaded as a library)", :upgrade)
           require path
         end
       rescue Exception => e
-        log_info("Proceeding with postload -- could not load #{path}: #{e} (#{e.class})\n  #{e.backtrace.join("\n  ")}")
+        log_info("Proceeding with postload -- could not load #{path}: #{e} (#{e.class})\n  #{e.backtrace.join("\n  ")}", :upgrade)
       else
         if defined?(einhorn_main)
-          log_info("Successfully loaded #{path}")
+          log_info("Successfully loaded #{path}", :upgrade)
           Einhorn::TransientState.preloaded = true
         else
-          log_info("Proceeding with postload -- loaded #{path}, but no einhorn_main method was defined")
+          log_info("Proceeding with postload -- loaded #{path}, but no einhorn_main method was defined", :upgrade)
         end
       end
     end

data/test/unit/einhorn/client.rb

@@ -59,12 +59,9 @@ class ClientTest < EinhornTestCase
 
     it "raises an error when deserializing invalid YAML" do
       invalid_serialized = "-%0A\t-"
-      expected = [ArgumentError]
-      expected << Psych::SyntaxError if defined?(Psych::SyntaxError) # 1.9
-
       begin
         Einhorn::Client::Transport.deserialize_message(invalid_serialized)
-      rescue *expected
+      rescue Einhorn::Client::Transport::ParseError
       end
     end
   end

data/test/unit/einhorn/event.rb

@@ -24,8 +24,8 @@ class EventTest < EinhornTestCase
     end
 
     it "selects on readable descriptors" do
-      sock1 = mock(:fileno => 4)
-      sock2 = mock(:fileno => 5)
+      sock1 = stub(:fileno => 4)
+      sock2 = stub(:fileno => 5)
 
       conn1 = Einhorn::Event::Connection.open(sock1)
       conn2 = Einhorn::Event::Connection.open(sock2)
@@ -41,8 +41,8 @@ class EventTest < EinhornTestCase
     end
 
     it "selects on writeable descriptors" do
-      sock1 = mock(:fileno => 4)
-      sock2 = mock(:fileno => 5)
+      sock1 = stub(:fileno => 4)
+      sock2 = stub(:fileno => 5)
 
       conn1 = Einhorn::Event::Connection.open(sock1)
       conn2 = Einhorn::Event::Connection.open(sock2)
@@ -61,8 +61,8 @@ class EventTest < EinhornTestCase
     end
 
     it "runs callbacks for ready selectables" do
-      sock1 = mock(:fileno => 4)
-      sock2 = mock(:fileno => 5)
+      sock1 = stub(:fileno => 4)
+      sock2 = stub(:fileno => 5)
 
       conn1 = Einhorn::Event::Connection.open(sock1)
       conn2 = Einhorn::Event::Connection.open(sock2)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: einhorn
 version: !ruby/object:Gem::Version
-  version: 0.4.9
+  version: 0.5.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-23 00:00:00.000000000 Z
+date: 2013-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -74,6 +74,7 @@ files:
 - .gitignore
 - .travis.yml
 - Gemfile
+- History.txt
 - LICENSE
 - README.md
 - README.md.in
@@ -129,12 +130,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 3689317997291483105
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 3689317997291483105
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -149,4 +156,3 @@ test_files:
 - test/unit/einhorn/command/interface.rb
 - test/unit/einhorn/event.rb
 - test/unit/einhorn/worker_pool.rb
-has_rdoc: