expedite 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8babab9c1e4ea0a7e6b07fe21738b029200d6f32cf1630552bce7fef3a7f6092
-  data.tar.gz: c24fd8f9e9cd7720c32d37329c67cccb089987b8a24df30bfedb45cdb20f48e8
+  metadata.gz: ece07855912fe5042042713eb78fecedc03cd06f5b495c0f17a1e7dab14ee99e
+  data.tar.gz: c94ea277e1a83968b187648d5917ed2e42181c81f3a3a04c4c6acafc1c52fbf5
 SHA512:
-  metadata.gz: 8beb17a4d477869134b0775b8437e15f51dbb9e51138b49355ea5fade24e7a771b07fdfec98f92c5bb28bf0cc876fa673befd3bd38848ca8c74fcda143996bd9
-  data.tar.gz: 3d810666ae634dfc032c99ee2b73d55f17790b552c85ab6dd4ba1e486c19374ac6e1d36fb0ff73badb1d5a807a5a1f1520ca7060dcda9e72ee2d306efed6cec1
+  metadata.gz: eb9688825664432ab5e7f5ef60ee855715eabcf6e25ee9242c6eca816c99bb7431eb36ad12b1124e11401dd961b4d68868077c41c9dd8f6b5bb20ffc854a1e09
+  data.tar.gz: baf6f7c68c3f98eccf8651947b72dd7e3150580d886015038e7ccd68e0c63798db82e8b6977c1f974a1c027112a233dc0fc227b002cb763151c3916fbda58a5f

data/README.md

@@ -16,7 +16,9 @@ This is the "parent" agent:
 ```
 Expedite.define do
   agent :parent do
-    $parent_var = 1
+    before(:serve) do |name|
+      $parent_var = name
+    end
   end
 end
 ```
@@ -26,8 +28,12 @@ matchers.
 
 ```
 Expedite.define do
-  agent "development/*", parent: :parent do |name|
-    $development_var = name
+  agent "development/*" do
+    self.parent = :parent
+
+    before(:serve) do |name|
+      $child_var = name
+    end
   end
 end
 ```
@@ -37,10 +43,12 @@ The following defines an `info` action.
 ```
 Expedite.define do
   action :info do
-    puts "     Process.pid = #{Process.pid}"
-    puts "    Process.ppid = #{Process.ppid}"
-    puts "     $parent_var = #{$parent_var}"
-    puts "$development_var = #{$development_var}"
+    {
+      "Process.pid" => Process.pid,
+      "Process.ppid" => Process.ppid,
+      "$parent_var" => $parent_var,
+      "$child_var" => $child_var,
+    }
   end
 end
 ```
@@ -55,7 +63,7 @@ the action; in that case, the return result is the exit code.
 ```
 require 'expedite'
 
-Expedite.agent("development/abc").invoke("info")
+puts Expedite.agent("development/abc").invoke("info")
 ```
 
 When you run `main.rb`, the following output is produced. Note that `$sleep_parent`

data/lib/expedite/action/block.rb

@@ -3,7 +3,7 @@ module Expedite
   module Action
     class Block
       attr_reader :runs_in
-      
+
       def initialize(runs_in: :application, &block)
         @runs_in = runs_in
         @block = block
@@ -13,6 +13,7 @@ module Expedite
         @block.call(*args)
       end
 
+      # Ignoring all streams
       def setup(_)
       end
     end

data/lib/expedite/action/boot.rb

@@ -2,12 +2,12 @@ module Expedite
   module Action
     class Boot
       def call(*args)
-        agent = args[0]
+        name = args[0]
 
         require "expedite/server/agent"
-        
+
         Expedite::Server::Agent.new(
-          agent: agent,
+          name: name,
           manager: UNIXSocket.for_fd(@child_socket.fileno),
           env: Expedite::Env.new(
             root: ENV['EXPEDITE_ROOT'],
@@ -16,9 +16,10 @@ module Expedite
         ).boot
       end
 
-      def setup(client)
-        @child_socket = client.recv_io
-        @log_file = client.recv_io
+      # STDOUT, STDERR, STDIN, log, child_socket
+      def setup(streams)
+        @log_file = streams[3]
+        @child_socket = streams[4]
       end
 
       def runs_in

data/lib/expedite/agents.rb

@@ -1,7 +1,10 @@
+require 'expedite/hooks'
 
 module Expedite
   # Definition of a Agent
   class Agent
+    include Hooks
+
     ##
     # Name of the parent agent. This allows you to create agents from
     # an existing agent.
@@ -16,18 +19,20 @@ module Expedite
     ##
     # [parent] Name of parent agent.
     # [keep_alive] Specifies if the agent should be automatically restarted if it is terminated. Defaults to false.
-    # [after_fork] Block is executed when agent is first preloaded.
-    def initialize(parent: nil, keep_alive: false, &after_fork)
+    def initialize(parent: nil, keep_alive: false)
       @parent = parent
       @keep_alive = keep_alive
-      @after_fork_proc = after_fork
     end
 
     ##
-    # Called when agent if first preloaded. This version calls the after_fork
-    # block provided in the initializer.
-    def after_fork(agent)
-      @after_fork_proc&.call(agent)
+    # Register a before event
+    # @params event [String] Allowed values: :run
+    def before(event, &block)
+      register_hook(:"before_#{event}", block)
+    end
+
+    def after(event, &block)
+      register_hook(:"after_#{event}", block)
     end
   end
 
@@ -54,8 +59,6 @@ module Expedite
     #
     # [matcher] Wildcard to match a name against.
     # [named_options] Agent options.
-    # [after_fork] Optional block that is called when
-    #              agent is preloaded.
     #
     # = Example
     #   Expedite::Agents.register('base' do |name|
@@ -64,8 +67,8 @@ module Expedite
     #   Expedite::Agents.register('development/abc', parent: 'base') do |name|
     #     puts "Agent #{name} started"
     #   end
-    def self.register(matcher, **named_options, &after_fork)
-      self.current.register(matcher.to_s, **named_options, &after_fork)
+    def self.register(matcher, **named_options)
+      self.current.register(matcher.to_s, **named_options)
     end
 
     ##
@@ -86,11 +89,10 @@ module Expedite
       ret.agent
     end
 
-    def register(matcher, **named_options, &after_fork)
-      @registrations << Registration.new(
-        matcher,
-        Agent.new(**named_options, &after_fork)
-      )
+    def register(matcher, **named_options)
+      agent = Agent.new(**named_options)
+      @registrations << Registration.new(matcher, agent)
+      agent
     end
 
     def reset

data/lib/expedite/client/agent_proxy.rb

@@ -4,23 +4,23 @@ require 'expedite/client/invoke'
 module Expedite
   module Client
     class AgentProxy
-      attr_accessor :env, :agent
+      attr_accessor :name, :env
 
       ##
       #
+      # @param name [String] Name of the agent
       # @param env [Expedite::Env] Environment
-      # @param agent [String] Name of the agent
-      def initialize(env:, agent:)
+      def initialize(name, env:)
+        self.name = name
         self.env = env
-        self.agent = agent
       end
 
       def exec(*args)
-        Client::Exec.new(env: env, agent: agent).call(*args)
+        Client::Exec.new(env: env, agent: name).call(*args)
       end
 
       def invoke(*args)
-        Client::Invoke.new(env: env, agent: agent).call(*args)
+        Client::Invoke.new(env: env, agent: name).call(*args)
       end
     end
   end

data/lib/expedite/client/base.rb

@@ -59,11 +59,11 @@ module Expedite
       end
 
       def run_command(client, agent, args)
+        @null_socket ||= File.open(File::NULL, "a")
         log "sending command"
 
-        agent.send_io STDOUT
-        agent.send_io STDERR
-        agent.send_io STDIN
+        # No child socket
+        agent.send_setup(@null_socket, env)
 
         agent.send_object({
           "args" => args,
@@ -85,7 +85,7 @@ module Expedite
           ## suspend_resume_on_tstp_cont(pid)
 
           ## forward_signals(application)
-          result = agent.recv_object
+          result = agent.recv_object(env)
           if result.key?("exception")
             e = result["exception"]
             log "got exception #{e}"

data/lib/expedite/hooks.rb

@@ -0,0 +1,46 @@
+module Expedite
+  module Hooks
+    ##
+    # Register a new hook with the given block to name
+    # @param name [String] Name of the hook
+    def register_hook(name, block)
+      return clear_hooks(name) if block.nil?
+
+      block = Array(block)
+      all_hooks[name].concat(block)
+    end
+
+    ##
+    # Clears all hooks for the specified name
+    # @param name [String] Name of the hook
+    def clear_hooks(name)
+      all_hooks[name] = []
+    end
+
+    ##
+    # Returns all hooks for the specified name
+    # @param name [String] Name of the hook
+    def hooks(name)
+      all_hooks[name]
+    end
+
+    ##
+    # Returne all hooks as a hash
+    def all_hooks
+      @all_hooks ||= Hash.new { |h, k| h[k] = [] }
+    end
+
+    ##
+    # Runs all Procs registered for the specified name
+    # @param name [String] Name of the hook
+    # @param @args Arguments passed to the Procs
+    def run_hook(name, *args)
+      hks = hooks(name)
+      return if hks.empty?
+
+      hks.each do |hook|
+        args.any? ? hook.call(*args) : hook.call
+      end
+    end
+  end
+end

data/lib/expedite/protocol.rb

@@ -8,6 +8,7 @@ module Expedite
 
       self.puts  data.bytesize.to_i
       self.write data
+      self.write "$"
     end
 
     ##
@@ -29,11 +30,38 @@ module Expedite
       self.send_object({"return" => obj}, env)
     end
 
-    def recv_object
+    def recv_object(env)
       len = self.gets.to_i
       data = self.read(len)
+      e = self.read(1)
+      env.log "recv_object len=#{len} data=... e=#{e}"
+      raise "Unexpected end #{e}" if e != "$"
       Marshal.load(data)
     end
+
+
+    ##
+    # Send sockets for setup.
+    # Linux seems to require all fds to be sent first
+    #
+    def send_setup(child_socket, env)
+      self.send_io STDOUT
+      self.send_io STDERR
+      self.send_io STDIN
+      self.send_io env.log_file
+      self.send_io child_socket
+    end
+
+    ##
+    # Receive setup sockets.
+    # Returns the child_socket
+    #
+    def recv_setup(env)
+      streams = 5.times.map { self.recv_io }
+      [STDOUT, STDERR, STDIN].zip(streams[0..2]).each { |a, b| a.reopen(b) }
+      env.log_file = streams[3]
+      return streams
+    end
   end
 end
 

data/lib/expedite/server/agent.rb

@@ -4,11 +4,11 @@ require 'pty'
 require 'set'
 require 'socket'
 require 'expedite/actions'
+require 'expedite/agents'
 require 'expedite/env'
 require 'expedite/failsafe_thread'
 require 'expedite/protocol'
 require 'expedite/signals'
-require 'expedite/agents'
 
 module Expedite
   def self.agent
@@ -33,11 +33,12 @@ module Expedite
     class Agent
       include Signals
 
-      attr_reader :agent
+      attr_reader :name
       attr_reader :manager, :env
 
-      def initialize(agent:, manager:, env:)
-        @agent      = agent
+      # @params name [String] Name of agent
+      def initialize(name:, manager:, env:)
+        @name         = name
         @manager      = manager
         @env          = env
         @mutex        = Mutex.new
@@ -56,8 +57,8 @@ module Expedite
         Signal.trap("TERM") { terminate }
 
         env.load_helper
-        eager_preload if false #if ENV.delete("SPRING_PRELOAD") == "1"
-        run
+
+        serve
       end
 
       def state(val)
@@ -75,8 +76,12 @@ module Expedite
         env.app_name
       end
 
+      def agent
+        @agent ||= Expedite::Agents.lookup(name)
+      end
+
       def log(message)
-        env.log "[application:#{agent}] #{message}"
+        env.log "[application:#{name}] #{message}"
       end
 
       def preloaded?
@@ -112,10 +117,10 @@ module Expedite
         with_pty { preload }
       end
 
-      def run
-        $0 = "expedite agent | #{app_name} | #{agent}"
+      def serve
+        $0 = "expedite agent | #{app_name} | #{name}"
 
-        Expedite::Agents.lookup(agent).after_fork(agent)
+        agent.run_hook(:before_serve, name)
 
         state :running
         manager.puts
@@ -124,36 +129,37 @@ module Expedite
           IO.select [manager, @interrupt.first]
 
           if terminating? || preload_failed?
+            agent.run_hook(:after_serve, name)
             exit
           else
-            serve manager.recv_io(UNIXSocket)
+            serve_request(manager.recv_io(UNIXSocket))
           end
         end
       end
 
-      def serve(client)
+      def serve_request(client)
         puts "got client"
         manager.puts
 
-        _stdout, stderr, _stdin = streams = 3.times.map { client.recv_io }
-        [STDOUT, STDERR, STDIN].zip(streams).each { |a, b| a.reopen(b) }
+        streams = client.recv_setup(env)
 
         preload unless preloaded?
 
-        args, env, method = client.recv_object.values_at("args", "env", "method")
-        log "serve #{args} using #{method}"
+        cargs, cenv, cmethod = client.recv_object(env).values_at("args", "env", "method")
+        log "serve #{cargs} using #{cmethod}"
 
-        exec_name = args.shift
+        exec_name = cargs.shift
         action    = Expedite::Actions.lookup(exec_name)
-        action.setup(client)
+        action.setup(streams)
+        # TODO: before(:request)
 
         connect_database # why are we connecting prior? is this for invoke?
-        pid = case method
+        pid = case cmethod
         when "invoke"
           # TODO: Invoke in a worker process instead of the preloader
-          serve_invoke(client, action, args, env)
+          serve_invoke(client, action, cargs, cenv)
         else
-          serve_fork(client, action, args, env)
+          serve_fork(client, action, cargs, cenv)
         end
 
         disconnect_database
@@ -186,25 +192,25 @@ module Expedite
       end
 
       # Returns pid of the current process
-      def serve_invoke(client, action, args, env)
+      def serve_invoke(client, action, cargs, cenv)
         begin
-          ret = action.call(*args)
+          ret = action.call(*cargs)
         rescue Exception => e
-          client.send_object({"exception" => e}, self.env)
+          client.send_exception(e, self.env)
         else
-          client.send_object({"return" => ret}, self.env)
+          client.send_return(ret, self.env)
         end
         Process.pid
       end
 
-      def serve_fork(client, action, args, env)
+      def serve_fork(client, action, cargs, cenv)
         fork do
           Process.setsid
           IGNORE_SIGNALS.each { |sig| trap(sig, "DEFAULT") }
           trap("TERM", "DEFAULT")
 
           # Load in the current env vars, except those which *were* changed when Spring started
-          env.each { |k, v| ENV[k] = v }
+          cenv.each { |k, v| ENV[k] = v }
 
           # requiring is faster, so if config.cache_classes was true in
           # the environment's config file, then we can respect that from
@@ -221,7 +227,7 @@ module Expedite
           shush_backtraces
 
           begin
-            ret = action.call(*args)
+            ret = action.call(*cargs)
           rescue => e
             client.send_exception(e, self.env)
           else

data/lib/expedite/server/agent_boot.rb

@@ -1,7 +1,7 @@
 require "expedite/server/agent"
 
 app = Expedite::Server::Agent.new(
-  agent: ENV['EXPEDITE_VARIANT'],
+  name: ENV['EXPEDITE_VARIANT'],
   manager: UNIXSocket.for_fd(3),
   env: Expedite::Env.new(
     root: ENV['EXPEDITE_ROOT'],

data/lib/expedite/server/agent_manager.rb

@@ -82,7 +82,7 @@ module Expedite
         end
       rescue Exception => e
         # NotImplementedError is an Exception, not StandardError
-        client.send_object({"exception" => e}, env)
+        client.send_exception(e, env)
         return Process.pid
       rescue Errno::ECONNRESET, Errno::EPIPE => e
         log "#{e} while reading from child; returning no pid"
@@ -95,9 +95,10 @@ module Expedite
         log "stopping"
         @state = :stopping
 
-        if pid
-          Process.kill('TERM', pid)
-          Process.wait(pid)
+        _pid = self.pid
+        if _pid
+          Process.kill('TERM', _pid)
+          Process.wait(_pid)
         end
       rescue Errno::ESRCH, Errno::ECHILD
         # Don't care
@@ -123,30 +124,31 @@ module Expedite
 
       # Creates a child that is forked from a parent
       def fork_child(preload = false)
-        @child, child_socket = UNIXSocket.pair
+        child_socket = nil
+        wr = nil
+        begin
+          @child, child_socket = UNIXSocket.pair(:STREAM)
 
-        # Compose command
-        wr, rd = UNIXSocket.pair
-        wr.send_io STDOUT
-        wr.send_io STDERR
-        wr.send_io STDIN
+          # Compose command
+          wr, rd = UNIXSocket.pair(:STREAM)
 
-        wr.send_object({
-          'args' => ['expedite/boot', name],
-          'env' => {},
-          'method' => "fork",
-        }, env)
+          wr.send_setup(child_socket, env)
 
-        wr.send_io child_socket
-        wr.send_io env.log_file
-        wr.close
+          wr.send_object({
+            'args' => ['expedite/boot', name],
+            'env' => {},
+            'method' => "fork",
+          }, env)
 
-        @pid = env.applications.with(parent) do |target|
-          target.run(rd)
-        end
+          @pid = env.applications.with(parent) do |target|
+            target.run(rd)
+          end
 
-        start_wait_thread(pid, child) if child.gets
-        child_socket.close
+          start_wait_thread(pid, child) if child.gets
+        ensure
+          wr&.close
+          child_socket&.close
+        end
       end
 
       # Creates a child that is started from scratch

data/lib/expedite/server/controller.rb

@@ -102,7 +102,7 @@ module Expedite
 
         # Corresponds to Client::Invoke#connect_to_agent
         app_client = client.recv_io
-        command    = client.recv_object
+        command    = client.recv_object(env)
 
         args, agent = command.values_at("args", "agent")
         cmd = args.first
@@ -114,9 +114,7 @@ module Expedite
             client.puts
 
             unix_socket = UNIXSocket.for_fd(app_client.fileno)
-            _stdout = unix_socket.recv_io
-            _stderr = unix_socket.recv_io
-            _stdin = unix_socket.recv_io
+            _ = unix_socket.recv_setup(env)
 
             client.puts Process.pid
 
@@ -124,7 +122,7 @@ module Expedite
             env.applications.pools.each do |k, pool|
               application_pids.concat(pool.all.map(&:pid))
             end
-            unix_socket.send_object({"return" => application_pids}, env)
+            unix_socket.send_return(application_pids, env)
 
             unix_socket.close
             client.close
@@ -142,11 +140,9 @@ module Expedite
             end
           rescue AgentNotFoundError => e
             unix_socket = UNIXSocket.for_fd(app_client.fileno)
-            _stdout = unix_socket.recv_io
-            _stderr = unix_socket.recv_io
-            _stdin = unix_socket.recv_io
+            _ = unix_socket.recv_setup(env)
 
-            args, env = unix_socket.recv_object.values_at("args", "env")
+            args, env = unix_socket.recv_object(env).values_at("args", "env")
 
             client.puts Process.pid
 

data/lib/expedite/syntax.rb

@@ -13,7 +13,9 @@ module Expedite
       end
 
       def agent(name, parent:nil, &block)
-        Expedite::Agents.register(name, parent: parent, &block)
+        agent = Expedite::Agents.register(name, parent: parent)
+        agent.instance_eval(&block) if !block.nil?
+        agent
       end
 
       def self.run(&block)
@@ -23,4 +25,4 @@ module Expedite
   end
 
   extend Syntax
-end
+end

data/lib/expedite/version.rb

@@ -1,3 +1,3 @@
 module Expedite
-  VERSION = '0.2.3'
+  VERSION = '0.2.4'
 end

data/lib/expedite.rb

@@ -9,6 +9,6 @@ module Expedite
   # @param env [Expedite::Env] Defaults to an environment pointing to the
   #                            current directory
   def self.agent(name, env: Env.new)
-    Client::AgentProxy.new(env: env, agent: name)
+    Client::AgentProxy.new(name, env: env)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: expedite
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Bing-Chang Lai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-12-30 00:00:00.000000000 Z
+date: 2023-02-08 00:00:00.000000000 Z
 dependencies: []
 description: Manages Ruby processes that can be used to spawn child processes faster.
 email: johnny.lai@me.com
@@ -37,6 +37,7 @@ files:
 - lib/expedite/errors.rb
 - lib/expedite/failsafe_thread.rb
 - lib/expedite/helper/rails.rb
+- lib/expedite/hooks.rb
 - lib/expedite/protocol.rb
 - lib/expedite/server/agent.rb
 - lib/expedite/server/agent_boot.rb