autorespawn 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 24740ded8fd2be27ffb5f7bd0d6e33f4b71f11f6
-  data.tar.gz: 32036b70931a245bc4e1c12b70446160e439123b
+  metadata.gz: 74818e88a4f45fa8d63abe962bfa92381816f8d9
+  data.tar.gz: a2df936e42159d3e7156f305710a4976ac75bf68
 SHA512:
-  metadata.gz: bf53297c07ebeb7edc339f60e961e9159da08b46275aa51cc8636002f08b37ec1a4b7410e4cd964b9be7c7fb6eb4540458427701739f464af7edc78f3f5cbee4
-  data.tar.gz: eba4eff1d2c3d1c3ee07e80c541584ceede4231e6b20952a4c324fcd9d1e0b3a19141328906a5a10299b15cffbca961c53435bcd76a435774a3505f5ce6e4019
+  metadata.gz: a999ef955f12ef443ca425db10394ae4996d46cf0c8bf129697a31404bb7f6c7164381c16a8ca11e802374db6249cb64b4c2cadb4b7becabadf2283c476ad247
+  data.tar.gz: 6a3f7b0ac6e2e160f6f4a8fc3db0f9aa866cc1e3a4c252005d92ba8f806c2734f7ca97c1f046c0856acbddac5e2f62acd4c9384e0c212bb8315bde60f3e7ae63

data/.travis.yml

@@ -3,3 +3,6 @@ rvm:
   - 2.1.6
   - 2.2.2
 before_install: gem install bundler -v 1.10.5
+script:
+    - bundler exec rake
+    - bundler exec rake test

data/Rakefile

@@ -7,4 +7,4 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList['test/**/*_test.rb']
 end
 
-task :default => :test
+task :default

data/lib/autorespawn.rb

@@ -181,7 +181,7 @@ def watch_yield
             end
             error_paths.merge(backtrace)
             if e.kind_of?(LoadError)
-                error_paths << e.path
+                error_paths << Pathname.new(e.path)
             end
         end
         required_paths.merge(currently_loaded_files - current)

data/lib/autorespawn/manager.rb

@@ -17,12 +17,14 @@ class Manager
         #   {#workers} and {#active_slaves}
         attr_reader :self_slave
         # @return [Integer] the number of processes allowed to work in parallel
-        attr_reader :parallel_level
+        attr_accessor :parallel_level
         # @return [Array<Slave>] declared worker processes, as a hash from
         #   the PID to a Slave object
         attr_reader :workers
         # @return [Hash<Slave>] list of active slaves
         attr_reader :active_slaves
+        # @return [Array<Slave>] list of slaves explicitely queued with {#queue}
+        attr_reader :queued_slaves
 
         # @!group Hooks
 
@@ -68,13 +70,14 @@ def on_slave_start(&block)
         # @!endgroup
 
         def initialize(name: nil, parallel_level: 1)
-            @parallel_level = parallel_level + 1
+            @parallel_level = parallel_level
             @workers   = Array.new
             @name = name
             @seed = ProgramID.for_self
 
             @self_slave = Self.new(name: name)
             @workers << self_slave
+            @queued_slaves = Array.new
             @active_slaves = Hash[self_slave.pid => self_slave]
         end
 
@@ -85,11 +88,27 @@ def register_seed_files(files, search_patch = seed.ruby_load_path, ignore_not_fo
             seed.register_files(files, search_path, ignore_not_found)
         end
 
+        # Check whether this manager has slaves
+        def has_slaves?
+            # There's always a worker for self
+            workers.size != 1
+        end
+
+        # The number of slaves registered
+        def slave_count
+            workers.size - 1
+        end
+
         # Tests whether this slave is registered as a worker on self
         def include?(slave)
             workers.include?(slave)
         end
 
+        # Tests whether this manager has some slaves that are active
+        def has_active_slaves?
+            active_slaves.size != 1
+        end
+
         # Tests whether this slave is currently active on self
         def active?(slave)
             active_slaves[slave.pid] == slave
@@ -126,43 +145,48 @@ def register_slave(slave)
             slave
         end
 
+        # Queue a slave for execution
+        def queue(slave)
+            queued_slaves << slave
+        end
+
         # @api private
         #
         # Collect information about the finished slaves
         #
         # @return [Array<Slave>] the slaves that finished
-        def collect_finished_slaves(wait: false)
+        def collect_finished_slaves
             finished_slaves = Array.new
-            waitpid_options =
-                if wait then []
-                else [Process::WNOHANG]
-                end
-
-            while finished_child = Process.waitpid2(-1, *waitpid_options)
-                pid, status = *finished_child
-                if slave = active_slaves.delete(pid)
-                    finished_slaves << slave
-                    slave.finished(status)
-                    slave.subcommands.each do |name, cmdline, spawn_options|
-                        add_slave(*cmdline, name: name, **spawn_options)
-                    end
-                    seed.merge!(slave.program_id)
-                    run_hook :on_slave_finished, slave
-                end
+            while finished_child = Process.waitpid2(-1, Process::WNOHANG)
+                finished_slaves << process_finished_slave(*finished_child)
             end
             finished_slaves
         rescue Errno::ECHILD
-            Array.new
+            finished_slaves
+        end
+
+        def process_finished_slave(pid, status)
+            return if !(slave = active_slaves.delete(pid))
+
+            slave.finished(status)
+            slave.subcommands.each do |name, cmdline, spawn_options|
+                add_slave(*cmdline, name: name, **spawn_options)
+            end
+            seed.merge!(slave.program_id)
+            run_hook :on_slave_finished, slave
+            slave
         end
 
         # Kill all active slaves
         #
         # @see clear
         def kill
-            active_slaves.each { |s| s.kill(join: false) }
-            while active_slaves != [self_slave]
-                collect_finished_slaves(wait: true)
+            active_slaves.each_value { |s| s.kill(join: false) }
+            while has_active_slaves?
+                finished_child = Process.waitpid2(-1)
+                process_finished_slave(*finished_child)
             end
+        rescue Errno::ECHILD
         end
 
         # Kill and remove all workers from this manager
@@ -191,13 +215,18 @@ def run
         end
 
         # Wait for children to terminate and spawns them when needed
-        def poll
+        def poll(autospawn: true)
             finished_slaves = collect_finished_slaves
             new_slaves = Array.new
-            while active_slaves.size < parallel_level
-                if slave_i = workers.index { |s| s.needs_spawn? }
+            while active_slaves.size < parallel_level + 1
+                if slave = queued_slaves.find { |s| !s.running? }
+                    queued_slaves.delete(slave)
+                elsif autospawn && (slave_i = workers.index { |s| s.needed? })
                     slave = workers.delete_at(slave_i)
                     @workers = workers[slave_i..-1] + workers[0, slave_i] + [slave]
+                end
+
+                if slave
                     slave.spawn
                     run_hook :__on_slave_start, slave
                     new_slaves << slave

data/lib/autorespawn/program_id.rb

@@ -8,7 +8,7 @@ class Autorespawn
     # It basically stores information about all the files that form this
     # program.
     class ProgramID
-        FileInfo = Struct.new :require_path, :path, :mtime, :size, :id
+        FileInfo = Struct.new :require_path, :path, :mtime, :size
 
         # Information about the files that form this program
         #
@@ -137,22 +137,18 @@ def id
             return @id if @id
 
             complete_id = files.keys.sort.map do |p|
-                files[p].id
+                "#{p}#{files[p].mtime}#{files[p].size}"
             end.join("")
             @id = Digest::SHA1.hexdigest(complete_id)
         end
 
         # Whether the state on disk is different than the state stored in self
         def changed?
-            files.each_value do |info|
+            files.each_value.any? do |info|
                 return true if !info.path.exist?
                 stat = info.path.stat
-                if stat.mtime != info.mtime || stat.size != info.size
-                    new_id = compute_file_id(info.path)
-                    return new_id != info.id
-                end
+                stat.mtime != info.mtime || stat.size != info.size
             end
-            false
         end
 
         def include?(path, search_path = ruby_load_path)
@@ -202,45 +198,7 @@ def ruby_load_path
         def file_info(path, search_path = ruby_load_path)
             resolved = resolve_file_path(path, search_path)
             stat = resolved.stat
-            id   = compute_file_id(resolved)
-            return FileInfo.new(path, resolved, stat.mtime, stat.size, id)
-        end
-
-        # @api private
-        #
-        # Compute the content ID of a text (code) file
-        def compute_text_file_id(file)
-            sanitized = file.readlines.map do |line|
-                # Remove unnecessary spaces
-                line = line.strip
-                line = line.gsub(/\s\s+/, ' ')
-                if !line.empty?
-                    line
-                end
-            end.compact
-            Digest::SHA1.hexdigest(sanitized.join("\n"))
-        end
-
-        # @api private
-        #
-        # Compute the content ID of a binary file
-        def compute_binary_file_id(file)
-            Digest::SHA1.hexdigest(file.read(enc: 'BINARY'))
-        end
-    
-        # Compute a SHA1 that is representative of the file's contents
-        #
-        # It does some whitespace cleanup, but is not meant to be super-robust
-        # to changes that are irrelevant to the end program
-        #
-        # @param [Pathname] file the path to the file
-        # @return [String] an ID string
-        def compute_file_id(file)
-            if file.extname == ".rb"
-                compute_text_file_id(file)
-            else
-                compute_binary_file_id(file)
-            end
+            return FileInfo.new(path, resolved, stat.mtime, stat.size)
         end
     end
 end

data/lib/autorespawn/self.rb

@@ -7,11 +7,12 @@ def initialize(*args, **options)
             @pid = Process.pid
         end
 
-        def needs_spawn?; false end
+        def needed?; false end
+        def needed!; end
         def spawn
             pid
         end
-        def kill
+        def kill(*, **)
         end
         def join
         end

data/lib/autorespawn/slave.rb

@@ -47,7 +47,7 @@ def initialize(*cmdline, name: nil, seed: ProgramID.new, env: Hash.new, **spawn_
             @name = name
             @program_id = seed.dup
             @cmdline    = cmdline
-            @needs_spawn = true
+            @needed = true
             @spawn_env     = env
             @spawn_options = spawn_options
             @subcommands = Array.new
@@ -85,7 +85,7 @@ def spawn
                 SLAVE_RESULT_ENV        => result_w.fileno.to_s)
 
             program_id.refresh
-            @needs_spawn = false
+            @needed = false
             pid = Kernel.spawn(env, *cmdline, initial_r => initial_r, result_w => result_w, **spawn_options)
             initial_r.close
             result_w.close
@@ -112,8 +112,24 @@ def spawn
 
         # Whether this slave would need to be spawned, either because it has
         # never be, or because the program ID changed
-        def needs_spawn?
-            @needs_spawn || !status || program_id.changed?
+        def needed?
+            !running? && (@needed || program_id.changed?)
+        end
+
+        # Marks this slave for execution
+        #
+        # Note that it will only be executed by the underlying {Manager} when
+        # (1) a slot is available and (2) it has stopped running
+        #
+        # This is usually not called directly, but through {Manager#queue} which
+        # also ensures that the slave gets in front of the execution queue
+        def needed!
+            @needed = true
+        end
+
+        # Resets {#needed?} to false
+        def not_needed!
+            @needed = false
         end
 
         # Whether the slave is running
@@ -175,7 +191,7 @@ def finished(status)
             modified = program_id.register_files(file_list)
             @program_id = program_id.slice(file_list)
             if !modified.empty?
-                @needs_spawn = true
+                needed!
             end
             result_r.close
             modified

data/lib/autorespawn/version.rb

@@ -1,3 +1,3 @@
 class Autorespawn
-    VERSION = "0.3.0"
+    VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: autorespawn
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Sylvain Joyeux
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-09-23 00:00:00.000000000 Z
+date: 2015-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hooks