autorespawn 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 407f820c7762cbbf5ae65f83ea578db5ffeb7e52
-  data.tar.gz: a5d66c0a60201f6956b149a7ecd8d2c42dfe15a2
+  metadata.gz: 24740ded8fd2be27ffb5f7bd0d6e33f4b71f11f6
+  data.tar.gz: 32036b70931a245bc4e1c12b70446160e439123b
 SHA512:
-  metadata.gz: 2f4024501a7eb0c401e71969ba806ccde95de5718a6eb2f04dc802936f776fc18621f04a586a69b2797128df4d052be6fd88a62a7a10516e41439967ce112f21
-  data.tar.gz: f47ae78d068236d2fec92cdb431cae60c23fdd4c3ae0e8972a009dee49cbd8da49a8ad933bc1354f01e4aa78645802e6c6813bd46101fcc6620307280918f682
+  metadata.gz: bf53297c07ebeb7edc339f60e961e9159da08b46275aa51cc8636002f08b37ec1a4b7410e4cd964b9be7c7fb6eb4540458427701739f464af7edc78f3f5cbee4
+  data.tar.gz: eba4eff1d2c3d1c3ee07e80c541584ceede4231e6b20952a4c324fcd9d1e0b3a19141328906a5a10299b15cffbca961c53435bcd76a435774a3505f5ce6e4019

data/.gitattributes

@@ -0,0 +1 @@
+*.rb diff=ruby

data/Gemfile

@@ -1,5 +1,6 @@
 source 'https://rubygems.org'
 
-gem 'flexmock', github: 'doudou/flexmock', branch: 'master'
+gem 'pry'
+gem 'pry-byebug'
 # Specify your gem's dependencies in autorespawn.gemspec
 gemspec

data/autorespawn.gemspec

@@ -25,6 +25,7 @@ EOD
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_dependency "hooks", ">= 0.4.1"
   spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", ">= 5.0", "~> 5.0"

data/lib/autorespawn.rb

@@ -1,9 +1,12 @@
 require 'set'
+require 'hooks'
+require 'autorespawn/hooks'
 require "autorespawn/version"
 require "autorespawn/exceptions"
 require "autorespawn/program_id"
 require "autorespawn/watch"
 require "autorespawn/slave"
+require "autorespawn/self"
 require "autorespawn/manager"
 
 # Automatically exec's the current program when one of the source file changes
@@ -25,31 +28,53 @@
 #   passed as-is to Kernel.spawn and Kernel.exec
 # @param options keyword options to pass to Kernel.spawn and Kernel.exec
 class Autorespawn
+    include Hooks
+    include Hooks::InstanceHooks
+
     INITIAL_STATE_FD = "AUTORESPAWN_AUTORELOAD"
 
     SLAVE_RESULT_ENV = 'AUTORESPAWN_SLAVE_RESULT_FD'
     SLAVE_INITIAL_STATE_ENV = 'AUTORESPAWN_SLAVE_INITIAL_STATE_FD'
 
+    # ID object
+    #
+    # An arbitrary object passed to {#initialize} or {#add_slave} to identify
+    # this process.
+    #
+    # @return [nil,Object]
+    def self.name
+        @name
+    end
     def self.slave_result_fd
         @slave_result_fd
     end
-    def self.slave_initial_state_fd
-        @slave_initial_state_fd
-    end
     def self.slave?
         !!slave_result_fd
     end
-
-    # Delete the envvars first, we really don't want them to leak
-    slave_initial_state_fd = ENV.delete(SLAVE_INITIAL_STATE_ENV)
-    slave_result_fd = ENV.delete(SLAVE_RESULT_ENV)
-
-    if slave_initial_state_fd
-        @slave_initial_state_fd = Integer(slave_initial_state_fd)
+    def self.initial_program_id
+        @initial_program_id
     end
-    if slave_result_fd
-        @slave_result_fd = Integer(slave_result_fd)
+
+    def self.read_child_state
+        # Delete the envvars first, we really don't want them to leak
+        slave_initial_state_fd = ENV.delete(SLAVE_INITIAL_STATE_ENV)
+        slave_result_fd = ENV.delete(SLAVE_RESULT_ENV)
+        if slave_initial_state_fd
+            slave_initial_state_fd = Integer(slave_initial_state_fd)
+            io = IO.for_fd(slave_initial_state_fd)
+            @name, @initial_program_id = Marshal.load(io)
+            io.close
+        end
+        if slave_result_fd
+            @slave_result_fd = Integer(slave_result_fd)
+        end
     end
+    read_child_state
+
+    # An arbitrary objcet that can be used to identify the processes/slaves
+    #
+    # @return [nil,Object]
+    attr_reader :name
 
     # The arguments that should be passed to Kernel.exec in standalone mode
     #
@@ -58,10 +83,26 @@ def self.slave?
     # @return [(Array,Hash)]
     attr_reader :process_command_line
 
-    # Set of callbacks called just before we respawn the process
+    # @!group Hooks
+
+    # @!method on_exception
+    #
+    # Register a callback that is called whenever an exception is rescued by
+    # {#watch_yield}
+    #
+    # @yieldparam [Exception] exception
+    define_hooks :on_exception
+
+    # @!method at_exit
+    #
+    # Register a callback that is called after the block passed to {#run} has
+    # been called, but before the process gets respawned. Meant to perform what
+    # hass been done in {#run} that should be cleaned before respawning.
     #
-    # @return [Array<#call>]
-    attr_reader :respawn_handlers
+    # @yieldparam [Exception] exception
+    define_hooks :at_respawn
+
+    # @!endgroup
 
     # @return [ProgramID] object currently known state of files makind this
     #   program
@@ -87,13 +128,15 @@ def self.slave?
     # In master/slave mode, the list of subcommands that the master should spawn
     attr_reader :subcommands
 
-    def initialize(*command, track_current: false, **options)
+    def initialize(*command, name: Autorespawn.name, track_current: false, **options)
         if command.empty?
             command = [$0, *ARGV]
         end
+        @name = name
+        @program_id = Autorespawn.initial_program_id ||
+            ProgramID.new
+
         @process_command_line = [command, options]
-        @respawn_handlers = Array.new
-        @program_id = ProgramID.new
         @exceptions = Array.new
         @required_paths = Set.new
         @error_paths = Set.new
@@ -104,18 +147,6 @@ def initialize(*command, track_current: false, **options)
         end
     end
 
-    # Returns true if there is an initial state dump
-    def has_initial_state?
-        !!Autorespawn.slave_initial_state_fd
-    end
-
-    # Loads the initial state from STDIN
-    def load_initial_state
-        io = IO.for_fd(Autorespawn.slave_initial_state_fd)
-        @program_id = Marshal.load(io)
-        io.close
-    end
-
     # Requires one file under the autorespawn supervision
     #
     # If the require fails, the call to {.run} will not execute its block,
@@ -134,8 +165,20 @@ def watch_yield
             raise
         rescue Exception => e
             new_exceptions << e
+            run_hook :on_exception, e
             exceptions << e
-            backtrace = e.backtrace_locations.map { |l| Pathname.new(l.absolute_path) }
+            # cross-drb exceptions are broken w.r.t. #backtrace_locations. It
+            # returns a string in their case. Since it happens only on
+            # exceptions that originate from the server (which means a broken
+            # Roby codepath), let's just ignore it
+            if !e.backtrace_locations.kind_of?(String)
+                backtrace = e.backtrace_locations.map { |l| Pathname.new(l.absolute_path) }
+            else
+                STDERR.puts "Caught what appears to be a cross-drb exception, which should not happen"
+                STDERR.puts e.message
+                e.backtrace.join("\n  ")
+                backtrace = Array.new
+            end
             error_paths.merge(backtrace)
             if e.kind_of?(LoadError)
                 error_paths << e.path
@@ -153,8 +196,8 @@ def slave?
     # Request that the master spawns these subcommands
     #
     # @raise [NotSlave] if the script is being executed in standalone mode
-    def add_slave(*cmdline, **spawn_options)
-        subcommands << [cmdline, spawn_options]
+    def add_slave(*cmdline, name: nil, **spawn_options)
+        subcommands << [name, cmdline, spawn_options]
     end
 
     # Create a pipe and dump the program ID state of the current program
@@ -164,7 +207,7 @@ def dump_initial_state(files)
         program_id.register_files(files)
 
         io = Tempfile.new "autorespawn_initial_state"
-        Marshal.dump(program_id, io)
+        Marshal.dump([name, program_id], io)
         io.flush
         io.rewind
         io
@@ -174,12 +217,6 @@ def currently_loaded_files
         $LOADED_FEATURES.map { |p| Pathname.new(p) } +
             caller_locations.map { |l| Pathname.new(l.absolute_path) }
     end
-        
-    # Declares a handler that should be called in a process, just before
-    # exec'ing a fresh process *if* the block has been executed
-    def at_respawn(&block)
-        respawn_handlers << block
-    end
 
     # Defines the exit code for this instance
     def exit_code(value = nil)
@@ -227,8 +264,8 @@ def run(&block)
                 raise ArgumentError, "cannot call #run with a block after using #add_slave"
             end
             manager = Manager.new
-            subcommands.each do |command, options|
-                manager.add_slave(*command, **options)
+            subcommands.each do |name, command, options|
+                manager.add_slave(*command, name: name, **options)
             end
             return manager.run
         end
@@ -236,10 +273,6 @@ def run(&block)
 
     # @api private
     def perform_work(all_files, &block)
-        if has_initial_state?
-            load_initial_state
-        end
-
         not_tracked = all_files.
             find_all do |p|
                 begin !program_id.include?(p)
@@ -250,14 +283,7 @@ def perform_work(all_files, &block)
         if not_tracked.empty? && !program_id.changed?
             if exceptions.empty?
                 did_yield = true
-                _, yield_exceptions = watch_yield(&block)
-                yield_exceptions.each do |e|
-                    backtrace = (e.backtrace || Array.new).dup
-                    first_line = backtrace.shift
-                    STDERR.puts "#{e.message}: #{first_line}"
-                    STDERR.puts "  #{e.backtrace.join("\n  ")}"
-                end
-
+                watch_yield(&block)
             end
 
             all_files = required_paths | error_paths
@@ -272,7 +298,7 @@ def perform_work(all_files, &block)
                 Watch.new(program_id).wait
             end
             if did_yield
-                respawn_handlers.each { |b| b.call }
+                run_hook :at_respawn
             end
         end
         all_files

data/lib/autorespawn/hooks.rb

@@ -0,0 +1,26 @@
+class Autorespawn
+    # Override of the global Hooks behaviour w.r.t. blocks. Blocks are evaluated
+    # in their definition context in Autorespawn instead of the default evaluation in
+    # the context of the receiver
+    module Hooks
+        include ::Hooks
+
+        def self.included(base)
+            base.class_eval do
+                extend Uber::InheritableAttr
+                extend ClassMethods
+                inheritable_attr :_hooks
+                self._hooks= HookSet.new
+            end
+        end
+
+        module ClassMethods
+            include ::Hooks::ClassMethods
+
+            def define_hooks(callback, scope: lambda { |c, s| s if !c.proc? })
+                super
+            end
+        end
+    end
+end
+

data/lib/autorespawn/manager.rb

@@ -1,7 +1,22 @@
 class Autorespawn
     # Manager of a bunch of autorespawn slaves
     class Manager
-        # @param [Integer] the number of processes allowed to work in parallel
+        include Hooks
+        include Hooks::InstanceHooks
+
+        # @return [ProgramID] a seed object that is passed to new slaves to
+        #   represent the currently known state of file, to avoid unnecessary
+        #   respawning
+        #
+        # @see add_seed_file
+        attr_reader :seed
+        # @return [Object] an object that is used to identify the manager itself
+        attr_reader :name
+        # @return [Self] an object that has the same API than [Slave] to
+        #   represent the manager's process itself. It is always included in
+        #   {#workers} and {#active_slaves}
+        attr_reader :self_slave
+        # @return [Integer] the number of processes allowed to work in parallel
         attr_reader :parallel_level
         # @return [Array<Slave>] declared worker processes, as a hash from
         #   the PID to a Slave object
@@ -9,17 +24,105 @@ class Manager
         # @return [Hash<Slave>] list of active slaves
         attr_reader :active_slaves
 
-        def initialize(parallel_level: 1)
-            @parallel_level = parallel_level
+        # @!group Hooks
+
+        # Register a callback for when a new slave is added by {#add_slave}
+        #
+        # @param [#call] block the callback
+        # @yieldparam [Slave] the new slave
+        def on_slave_new(&block)
+            __on_slave_new(&block)
+            workers.each do |w|
+                block.call(w)
+            end
+        end
+        define_hooks :__on_slave_new
+
+        # Register a callback that should be called when a new slave has been
+        # spawned by {#poll}
+        #
+        # @param [#call] block the callback
+        # @yieldparam [Slave] the newly started slave
+        def on_slave_start(&block)
+            __on_slave_start(&block)
+            active_slaves.each_value do |w|
+                block.call(w)
+            end
+        end
+        define_hooks :__on_slave_start
+        
+        # @!method on_slave_finished
+        #
+        # Hook called when a slave finishes
+        #
+        # @yieldparam [Slave] the slave
+        define_hooks :on_slave_finished
+        
+        # @!method on_slave_removed
+        #
+        # Hook called when a slave has been removed from this manager
+        #
+        # @yieldparam [Slave] the slave
+        define_hooks :on_slave_removed
+
+        # @!endgroup
+
+        def initialize(name: nil, parallel_level: 1)
+            @parallel_level = parallel_level + 1
             @workers   = Array.new
-            @active_slaves = Hash.new
+            @name = name
+            @seed = ProgramID.for_self
+
+            @self_slave = Self.new(name: name)
+            @workers << self_slave
+            @active_slaves = Hash[self_slave.pid => self_slave]
+        end
+
+        # Add files to {#seed}
+        #
+        # (see ProgramID#register_files)
+        def register_seed_files(files, search_patch = seed.ruby_load_path, ignore_not_found: true)
+            seed.register_files(files, search_path, ignore_not_found)
+        end
+
+        # Tests whether this slave is registered as a worker on self
+        def include?(slave)
+            workers.include?(slave)
+        end
+
+        # Tests whether this slave is currently active on self
+        def active?(slave)
+            active_slaves[slave.pid] == slave
         end
 
         # Spawns a worker, i.e. a program that will perform the intended work
         # and report the program state
-        def add_slave(*cmdline, **spawn_options)
-            slave = Slave.new(*cmdline, **spawn_options)
+        #
+        # @param [Object] name an arbitrary object that can be used for
+        #   reporting / tracking
+        def add_slave(*cmdline, name: nil, **spawn_options)
+            slave = Slave.new(*cmdline, name: name, seed: seed, **spawn_options)
+            register_slave(slave)
+            slave
+        end
+
+        # Remove a worker from this manager
+        #
+        # @raise [ArgumentError] if the slave is still running
+        def remove_slave(slave)
+            if active?(slave)
+                raise ArgumentError, "#{slave} is still running"
+            end
+            workers.delete(slave)
+            run_hook :on_slave_removed, slave
+        end
+
+        # @api private
+        #
+        # Registers a slave
+        def register_slave(slave)
             workers << slave
+            run_hook :__on_slave_new, slave
             slave
         end
 
@@ -28,13 +131,23 @@ def add_slave(*cmdline, **spawn_options)
         # Collect information about the finished slaves
         #
         # @return [Array<Slave>] the slaves that finished
-        def collect_finished_slaves
+        def collect_finished_slaves(wait: false)
             finished_slaves = Array.new
-            while finished_child = Process.waitpid2(-1, Process::WNOHANG)
+            waitpid_options =
+                if wait then []
+                else [Process::WNOHANG]
+                end
+
+            while finished_child = Process.waitpid2(-1, *waitpid_options)
                 pid, status = *finished_child
                 if slave = active_slaves.delete(pid)
                     finished_slaves << slave
                     slave.finished(status)
+                    slave.subcommands.each do |name, cmdline, spawn_options|
+                        add_slave(*cmdline, name: name, **spawn_options)
+                    end
+                    seed.merge!(slave.program_id)
+                    run_hook :on_slave_finished, slave
                 end
             end
             finished_slaves
@@ -42,6 +155,28 @@ def collect_finished_slaves
             Array.new
         end
 
+        # Kill all active slaves
+        #
+        # @see clear
+        def kill
+            active_slaves.each { |s| s.kill(join: false) }
+            while active_slaves != [self_slave]
+                collect_finished_slaves(wait: true)
+            end
+        end
+
+        # Kill and remove all workers from this manager
+        #
+        # @see kill
+        def clear
+            kill
+            workers.dup.each do |w|
+                if w != self_slave
+                    remove_slave(w)
+                end
+            end
+        end
+
         def run
             while true
                 poll
@@ -64,6 +199,7 @@ def poll
                     slave = workers.delete_at(slave_i)
                     @workers = workers[slave_i..-1] + workers[0, slave_i] + [slave]
                     slave.spawn
+                    run_hook :__on_slave_start, slave
                     new_slaves << slave
                     active_slaves[slave.pid] = slave
                 else

data/lib/autorespawn/program_id.rb

@@ -19,6 +19,21 @@ def initialize
             @files = Hash.new
         end
 
+        def initialize_copy(old)
+            super
+            @files = @files.dup
+        end
+
+        # Merge the information contained in another ProgramID object into self
+        #
+        # @param [ProgramID] id the object whose information we should merge
+        # @return self
+        def merge!(id)
+            @files.merge!(id.files)
+            @id = nil
+            self
+        end
+
         # Compute ID information abou thte current Ruby process
         def self.for_self
             id = ProgramID.new
@@ -40,18 +55,20 @@ def clear
         # 
         # @return [void]
         def register_loaded_features
-            search_path = ruby_load_path
-            $LOADED_FEATURES.each do |file|
-                # enumerator.so is listed in $LOADED_FEATURES but is not present
-                # on disk ... no idea
-                begin
-                    begin
-                        register_file(Pathname.new(file))
-                    rescue FileNotFound => e
-                        STDERR.puts "WARN: could not find #{e.path} in ruby search path, ignored"
-                    end
-                end
+            loaded_features = $LOADED_FEATURES.map do |file|
+                Pathname.new(file)
             end
+            register_files(loaded_features)
+        end
+
+        # Resolve a file list into absolute paths
+        def resolve_file_list(files, search_path = ruby_load_path, ignore_not_found: true)
+            files.map do |path|
+                begin resolve_file_path(path, search_path)
+                rescue FileNotFound
+                    raise if !ignore_not_found
+                end
+            end.compact
         end
 
         # Register a set of files
@@ -63,16 +80,20 @@ def register_loaded_features
         # @return [Boolean] whether the program ID has been modified
         def register_files(files, search_path = ruby_load_path, ignore_not_found: true)
             modified = Array.new
-            files.each do |path|
-                begin
-                    if full_path = register_file(path, search_path)
-                        modified << full_path
-                    end
-                rescue FileNotFound
-                    raise if !ignore_not_found
-                end
+            files = resolve_file_list(files, search_path, ignore_not_found: ignore_not_found)
+            files.find_all do |path|
+                register_file(path, search_path)
             end
-            modified
+        end
+
+        # Removes any file in self that is not in the given file list and
+        # returns the result
+        def slice(files, search_path = ruby_load_path, ignore_not_found: true)
+            result = dup
+            files = resolve_file_list(files, search_path, ignore_not_found: ignore_not_found).
+                to_set
+            result.files.delete_if { |k, _| !files.include?(k) }
+            result
         end
 
         # Registers file information for one file

data/lib/autorespawn/self.rb

@@ -0,0 +1,25 @@
+class Autorespawn
+    # A Slave-compatible object that represents the manager's process itself
+    class Self < Slave
+        def initialize(*args, **options)
+            super
+
+            @pid = Process.pid
+        end
+
+        def needs_spawn?; false end
+        def spawn
+            pid
+        end
+        def kill
+        end
+        def join
+        end
+        def running?
+            true
+        end
+        def finished?
+            false
+        end
+    end
+end

data/lib/autorespawn/slave.rb

@@ -5,6 +5,12 @@ class Autorespawn
     # Slaves have two roles: the one of discovery (what are the commands that
     # need to be started) and the one of 
     class Slave
+        # The slave's name
+        #
+        # It is an arbitrary object useful for reporting/tracking
+        #
+        # @return [Object]
+        attr_reader :name
         # The currently known program ID
         attr_reader :program_id
         # The command line of the subprocess
@@ -33,8 +39,13 @@ class Slave
         # @return [String] the result data as received
         attr_reader :result_buffer
 
-        def initialize(*cmdline, env: Hash.new, **spawn_options)
-            @program_id = ProgramID.new
+        # @param [Object] name an arbitrary object that can be used for
+        #   reporting / tracking reasons
+        # @param [ProgramID] seed a seed object with some relevant files already
+        #   registered, to avoid respawning the slave unnecessarily.
+        def initialize(*cmdline, name: nil, seed: ProgramID.new, env: Hash.new, **spawn_options)
+            @name = name
+            @program_id = seed.dup
             @cmdline    = cmdline
             @needs_spawn = true
             @spawn_env     = env
@@ -52,6 +63,13 @@ def inspect
 
         def to_s; inspect end
 
+        # Register files on the program ID
+        #
+        # (see ProgramID#register_files)
+        def register_files(files, search_path = program_id.ruby_load_path, ignore_not_found: true)
+            program_id.register_files(files, search_path, ignore_not_found: ignore_not_found)
+        end
+
         # Start the slave
         #
         # @return [Integer] the slave's PID
@@ -71,7 +89,7 @@ def spawn
             pid = Kernel.spawn(env, *cmdline, initial_r => initial_r, result_w => result_w, **spawn_options)
             initial_r.close
             result_w.close
-            Marshal.dump(program_id, initial_w)
+            Marshal.dump([name, program_id], initial_w)
 
             @pid = pid
             @status = nil
@@ -155,6 +173,7 @@ def finished(status)
                 @success = false
             end
             modified = program_id.register_files(file_list)
+            @program_id = program_id.slice(file_list)
             if !modified.empty?
                 @needs_spawn = true
             end

data/lib/autorespawn/version.rb

@@ -1,3 +1,3 @@
 class Autorespawn
-    VERSION = "0.2.1"
+    VERSION = "0.3.0"
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: autorespawn
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Sylvain Joyeux
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-09-19 00:00:00.000000000 Z
+date: 2015-09-23 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: hooks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +136,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitattributes"
 - ".gitignore"
 - ".travis.yml"
 - Gemfile
@@ -131,8 +146,10 @@ files:
 - autorespawn.gemspec
 - lib/autorespawn.rb
 - lib/autorespawn/exceptions.rb
+- lib/autorespawn/hooks.rb
 - lib/autorespawn/manager.rb
 - lib/autorespawn/program_id.rb
+- lib/autorespawn/self.rb
 - lib/autorespawn/slave.rb
 - lib/autorespawn/version.rb
 - lib/autorespawn/watch.rb
@@ -161,3 +178,4 @@ signing_key:
 specification_version: 4
 summary: functionality to respawn a Ruby program when its source changes
 test_files: []
+has_rdoc: