supervisor_machines 0.1.1

data/lib/supervisor_machines/supervisor.rb

@@ -0,0 +1,320 @@
+# frozen_string_literal: true
+
+module SupervisorMachines
+  # A supervision tree over threads.
+  #
+  # Which children a failure takes down, and whether restarting is still worth
+  # trying, belongs entirely to {Decider}. This class owns the lifecycle:
+  # spawning children, noticing when one stops, waiting out the backoff
+  # +chrono_machines+ hands it, and taking the tree down in reverse start order
+  # inside a +deadline_machines+ budget.
+  #
+  # A tree is +registering+ until {#start} fixes its shape, +running+ while it
+  # watches its children, and ends either +stopped+ or +failed+. Both endings
+  # are final: {#start} refuses a tree that is not registering, and a +failed+
+  # tree answers every {#stop} with {Escalated}.
+  #
+  #   tree = SupervisorMachines::Supervisor.new(strategy: :one_for_all, restarts: 3, window: 60.0)
+  #   tree.supervise(:cache) { Thread.new { cache.run } }
+  #   tree.supervise(:warmup, policy: :transient) { Thread.new { warm } }
+  #   tree.start
+  #   tree.stop(grace: 5.0)
+  class Supervisor
+    # A registered child and whichever thread currently runs it. +epoch+ rises
+    # every time the child is taken down, so a thread the tree has already
+    # replaced neither reports its exit nor takes the child's place when its
+    # starter returns late. +done+ marks a child the tree has dropped, which no
+    # group restart brings back.
+    Child = Struct.new(:name, :policy, :starter, :thread, :epoch, :done)
+
+    # Wakes the monitor when the tree is being taken down.
+    STOP = Object.new
+    private_constant :STOP
+
+    attr_reader :strategy
+
+    # +restarts+ within +window+ seconds is the restart intensity; past it the
+    # tree escalates. +backoff+ is asked for the delay between a restart
+    # decision and the respawn; its attempt budget has to be one wider than the
+    # intensity, because the core escalates one restart after the last one it
+    # tolerates.
+    def initialize(strategy: :one_for_one, restarts: DEFAULT_RESTARTS,
+                   window: DEFAULT_WINDOW_NS / NS_PER_SECOND.to_f, backoff: nil)
+      raise ArgumentError, "unknown strategy: #{strategy.inspect}" unless STRATEGIES.include?(strategy)
+      raise ArgumentError, "at most #{Decider::MAX_RESTARTS} restarts" if restarts > Decider::MAX_RESTARTS
+
+      @strategy = strategy
+      @restarts = restarts
+      @window = window
+      @backoff = backoff || ChronoMachines::Executor.new(max_attempts: restarts + 1)
+      @slots = []
+      @events = Thread::Queue.new
+      @mutex = Thread::Mutex.new
+      @halt = Thread::ConditionVariable.new
+      @state = :registering
+      @failed = nil
+    end
+
+    # Registers a child under +name+. The block is its starter: it returns the
+    # +Thread+ that runs the child, and is called again for every restart.
+    def supervise(name, policy: :permanent, &starter)
+      raise ArgumentError, "unknown policy: #{policy.inspect}" unless POLICIES.include?(policy)
+
+      @mutex.synchronize do
+        raise Error, "a #{@state} supervisor cannot take on a child" unless @state == :registering
+
+        @slots << Child.new(name, policy, starter, nil, 0, false)
+      end
+      self
+    end
+
+    # Fixes the shape of the tree, starts every child in registration order and
+    # begins watching them. A starter that will not run takes the children
+    # already up back down, leaves the tree failed, and raises what the starter
+    # raised.
+    #
+    # Raises {Error} for a tree that is past registering.
+    def start
+      @mutex.synchronize do
+        raise Error, "a #{@state} supervisor cannot be started" unless @state == :registering
+
+        @decider = Decider.new(children: @slots.size, strategy: @strategy,
+                               restarts: @restarts, window_ns: to_ns(@window))
+        @state = :running
+        @monitor = Thread.new { monitor }
+      end
+      failure = launch
+      return self unless failure
+
+      rollback
+      raise failure
+    end
+
+    # Child names, in start order.
+    def children
+      @mutex.synchronize { @slots.map(&:name) }
+    end
+
+    # Restarts still inside the intensity window.
+    def restarts
+      now = monotonic_ns
+      @mutex.synchronize { @decider ? @decider.recent_restarts(now) : 0 }
+    end
+
+    # True once the tree has escalated, and true from then on.
+    def terminated?
+      @mutex.synchronize { @state == :failed }
+    end
+
+    # Takes every running child down in reverse start order, giving the whole
+    # shutdown +grace+ seconds, or all the time it needs when +grace+ is nil.
+    # A tree still registering has nothing to take down and stays registering.
+    #
+    # Raises {Escalated} for a tree that gave up, every time it is called.
+    def stop(grace: nil)
+      entered = @mutex.synchronize do
+        state = @state
+        @state = :stopped if state == :running
+        @halt.broadcast
+        state
+      end
+      return self if entered == :registering
+
+      @events.push(STOP) if entered == :running
+      failures = nil
+      DeadlineMachines.with_deadline(grace) do |deadline|
+        failures = shutdown
+        @monitor.kill unless @monitor.join(deadline.remaining)
+      end
+      raise Escalated, escalation if terminated?
+      raise failures.first unless failures.empty?
+
+      self
+    end
+
+    # Seconds a supervisor taking itself down waits for a killed child, since
+    # neither the monitor thread nor a failing {#start} has a caller budget to
+    # read.
+    ESCALATION_GRACE = 5.0
+
+    private
+
+    # Brings every child up in registration order, returning what a starter
+    # raised, or nil once the whole tree is up.
+    def launch
+      @slots.each_index { |index| spawn(index) }
+      nil
+    rescue Exception => e
+      @mutex.synchronize do
+        @state = :failed
+        @failed = unstarted(e)
+      end
+      e
+    end
+
+    # Takes down whatever a start brought up before it failed, the children and
+    # the monitor watching them, inside one budget.
+    def rollback
+      DeadlineMachines.with_deadline(ESCALATION_GRACE) do |deadline|
+        shutdown
+        reap(@monitor, deadline.remaining)
+      end
+    end
+
+    def running?
+      @state == :running
+    end
+
+    def monitor
+      Thread.current.report_on_exception = false
+      loop do
+        event = @events.pop
+        break if event.equal?(STOP)
+
+        handle(*event)
+      end
+    end
+
+    # One child stopped. The core says what that means; this carries it out.
+    def handle(index, epoch, ending)
+      now = monotonic_ns
+      decision = @mutex.synchronize do
+        child = @slots[index]
+        next unless running? && child.epoch == epoch
+
+        child.thread = nil
+        child.done = true if child.policy == :temporary
+        @decider.stopped(index: index, policy: child.policy, exit: ending, now_ns: now)
+      end
+
+      case decision
+      when Range then restart(decision)
+      when :escalate then escalate
+      end
+    end
+
+    # Retires the whole span in reverse start order and waits for it to go,
+    # waits out the backoff without holding the tree, then brings the span back
+    # in index order.
+    def restart(span)
+      retiring = @mutex.synchronize { span.reverse_each.filter_map { |index| retire(@slots[index]) } }
+      DeadlineMachines.with_deadline(ESCALATION_GRACE) do |deadline|
+        retiring.each { |thread| reap(thread, deadline.remaining) }
+      end
+      return escalate(abandoned) if retiring.any?(&:alive?)
+
+      failure = respawn(span)
+      escalate(failure) if failure
+    end
+
+    # The reason a respawn could not happen, or nil once the span is running.
+    def respawn(span)
+      delay = @backoff.next_delay(attempt: restarts)
+      @mutex.synchronize { @halt.wait(@mutex, delay) if delay && running? }
+      span.each { |index| spawn(index) }
+      nil
+    rescue Exception => e
+      unstarted(e)
+    end
+
+    def escalate(reason = escalation)
+      @mutex.synchronize do
+        @state = :failed
+        @failed = reason
+      end
+      shutdown(ESCALATION_GRACE)
+      raise Escalated, reason
+    end
+
+    # Reverse start order, each child killed and then waited for inside one
+    # budget shared by the whole tree. Touches no tree state, so a shutdown
+    # under way in the monitor never holds up the one in {#stop}. Returns what
+    # the children raised on their way out, having stopped all of them.
+    def shutdown(grace = DeadlineMachines.remaining)
+      failures = []
+      DeadlineMachines.with_deadline(grace) do |deadline|
+        @slots.reverse_each do |child|
+          thread = child.thread
+          next unless thread
+
+          failures << reap(thread, deadline.remaining)
+        end
+      end
+      failures.compact
+    end
+
+    # Kills the thread and waits out the rest of the budget for it, reporting
+    # whatever it raised on the way out. A thread that outlives the budget is
+    # still alive afterwards.
+    def reap(thread, remaining)
+      thread.kill
+      thread.join(remaining)
+      nil
+    rescue Exception => e
+      e
+    end
+
+    # Calls the child's starter outside the lock, under the generation the tree
+    # is in when the call goes out. A child the tree has dropped, and every
+    # child of a tree that is no longer running, is left alone.
+    def spawn(index)
+      child = @slots[index]
+      generation = @mutex.synchronize { child.epoch if running? && !child.done }
+      install(index, generation, child.starter.call) if generation
+    end
+
+    # Hands the started thread to the child, unless the tree has moved past the
+    # generation that asked for it, in which case the thread is killed instead
+    # of left running with nothing watching it.
+    def install(index, generation, thread)
+      installed = @mutex.synchronize do
+        child = @slots[index]
+        next false unless running? && child.epoch == generation
+
+        child.thread = thread
+        watch(index, generation, thread)
+        true
+      end
+      reap(thread, ESCALATION_GRACE) unless installed
+    end
+
+    # Invalidates the watcher following the child and hands its thread back to
+    # be reaped outside the lock.
+    def retire(child)
+      child.epoch += 1
+      thread = child.thread
+      child.thread = nil
+      thread
+    end
+
+    # One thread per running child, blocked on it, reporting how it ended.
+    def watch(index, epoch, thread)
+      Thread.new do
+        thread.join
+        @events.push([index, epoch, :normal])
+      rescue Exception
+        @events.push([index, epoch, :abnormal])
+      end
+    end
+
+    def escalation
+      @failed || "#{@strategy} supervisor escalated: more than #{@restarts} restarts in #{@window} seconds"
+    end
+
+    def abandoned
+      "#{@strategy} supervisor escalated: a retired child would not stop within #{ESCALATION_GRACE} seconds"
+    end
+
+    def unstarted(error)
+      "#{@strategy} supervisor escalated: #{error.message}"
+    end
+
+    def monotonic_ns
+      DeadlineMachines.clock.now_ns
+    end
+
+    def to_ns(seconds)
+      DeadlineMachines::Deadline.to_ns(seconds)
+    end
+  end
+end

data/lib/supervisor_machines/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SupervisorMachines
+  VERSION = '0.1.1'
+end

data/lib/supervisor_machines.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'chrono_machines'
+require 'deadline_machines'
+
+require_relative 'supervisor_machines/version'
+
+# Decider is defined by the native extension, which binds the
+# `supervisor-machines` crate; there is no pure-Ruby implementation to fall
+# back to.
+require 'supervisor_machines_native/supervisor_machines_native'
+
+require_relative 'supervisor_machines/errors'
+require_relative 'supervisor_machines/supervisor'

data/sig/supervisor_machines.rbs

@@ -0,0 +1,173 @@
+# OTP-style supervision over Ruby threads.
+#
+# `Decider` is defined by the native extension, which binds the
+# `supervisor-machines` crate: it decides which children a failure takes down
+# and when restarting has stopped being worth it. `Supervisor` owns everything
+# the core refuses to touch, which is every thread.
+module SupervisorMachines
+  VERSION: String
+
+  # Nanoseconds in a second. Seconds are what the Ruby surface takes; the core
+  # is fed nanosecond integers.
+  NS_PER_SECOND: Integer
+
+  # Restarts OTP tolerates by default before escalating.
+  DEFAULT_RESTARTS: Integer
+
+  # The window OTP measures those restarts over, five seconds.
+  DEFAULT_WINDOW_NS: Integer
+
+  # Which siblings a failure takes down with it.
+  type strategy = :one_for_one | :one_for_all | :rest_for_one
+
+  # When a child is worth restarting.
+  type policy = :permanent | :transient | :temporary
+
+  # How a child's run ended.
+  type exit_kind = :normal | :abnormal
+
+  # What the core decided: leave it stopped, restart this inclusive range of
+  # child indices, or give up.
+  type decision = :ignore | :escalate | Range[Integer]
+
+  # Every strategy, in the order the core declares them. Anything else raises
+  # `ArgumentError`.
+  STRATEGIES: Array[strategy]
+
+  # Every policy, in the order the core declares them. Anything else raises
+  # `ArgumentError`.
+  POLICIES: Array[policy]
+
+  class Error < StandardError
+  end
+
+  # Raised once a supervisor has spent its restart intensity.
+  class Escalated < Error
+  end
+
+  # The core's decision machine. A child count and a restart history are part
+  # of its Rust type, so the count given here is dispatched to one of
+  # `SUPPORTED_CHILDREN` monomorphised shapes.
+  class Decider
+    # Restart timestamps every shape keeps.
+    HISTORY: Integer
+
+    # The child counts a decider can be built for. Anything else raises
+    # `ArgumentError`.
+    SUPPORTED_CHILDREN: Array[Integer]
+
+    def self.new: (
+      children: Integer,
+      strategy: strategy,
+      restarts: Integer,
+      window_ns: Integer
+    ) -> instance
+
+    # Feeds one stopped child to the core and returns what it decided.
+    def stopped: (
+      index: Integer,
+      policy: policy,
+      exit: exit_kind,
+      now_ns: Integer
+    ) -> decision
+
+    def children: () -> Integer
+
+    def strategy: () -> strategy
+
+    # True once the core has escalated, and true from then on.
+    def terminated?: () -> bool
+
+    # Restarts still inside the intensity window as of `now_ns`.
+    def recent_restarts: (Integer) -> Integer
+
+    def inspect: () -> String
+  end
+
+  # A supervision tree over threads.
+  class Supervisor
+    # Where a tree is in its lifecycle: registering children, running them, or
+    # ended — `stopped` by its caller, `failed` by giving up.
+    type state = :registering | :running | :stopped | :failed
+
+    # Seconds a supervisor taking itself down waits for a killed child.
+    ESCALATION_GRACE: Float
+
+    # Anything that can price the wait between a restart decision and the
+    # respawn, which is `ChronoMachines::Executor` unless told otherwise.
+    interface _Backoff
+      def next_delay: (attempt: Integer) -> Numeric?
+    end
+
+    # A registered child and whichever thread currently runs it.
+    class Child < ::Struct[untyped]
+      attr_accessor name: untyped
+      attr_accessor policy: policy
+      attr_accessor starter: ^() -> Thread
+      attr_accessor thread: Thread?
+      attr_accessor epoch: Integer
+      attr_accessor done: bool
+    end
+
+    attr_reader strategy: strategy
+
+    # `restarts` within `window` seconds is the restart intensity.
+    def initialize: (
+      ?strategy: strategy,
+      ?restarts: Integer,
+      ?window: Float,
+      ?backoff: _Backoff?
+    ) -> void
+
+    # Registers a child under `name`. The block returns the `Thread` that runs
+    # it, and is called again for every restart.
+    def supervise: (untyped name, ?policy: policy) { () -> Thread } -> self
+
+    # Fixes the shape of the tree and starts every child in registration
+    # order. Raises `ArgumentError` for a child count the core has no shape
+    # for, `Error` for a tree that is past registering, and whatever a starter
+    # raised after taking the children already up back down.
+    def start: () -> self
+
+    # Child names, in start order.
+    def children: () -> Array[untyped]
+
+    # Restarts still inside the intensity window.
+    def restarts: () -> Integer
+
+    def terminated?: () -> bool
+
+    # Takes every running child down in reverse start order within `grace`
+    # seconds, or all the time it needs when `grace` is nil. A tree still
+    # registering has nothing to take down. Raises `Escalated` for a tree that
+    # gave up, every time it is called.
+    def stop: (?grace: Float?) -> self
+
+    private
+
+    # What a starter raised, after the children already up have been taken
+    # back down, or nil once the whole tree is up.
+    def launch: () -> Exception?
+    def rollback: () -> void
+    def running?: () -> bool
+    def monitor: () -> void
+    def handle: (Integer index, Integer epoch, exit_kind ending) -> void
+    def restart: (Range[Integer] span) -> void
+    def respawn: (Range[Integer] span) -> String?
+    def escalate: (?String reason) -> void
+    def shutdown: (?Float grace) -> Array[Exception]
+    def reap: (Thread thread, Float? remaining) -> Exception?
+    def spawn: (Integer index) -> void
+
+    # Installs a started thread under the generation that asked for it, killing
+    # it when the tree has moved past that generation.
+    def install: (Integer index, Integer generation, Thread thread) -> void
+    def retire: (Child child) -> Thread?
+    def watch: (Integer index, Integer epoch, Thread thread) -> void
+    def escalation: () -> String
+    def abandoned: () -> String
+    def unstarted: (Exception error) -> String
+    def monotonic_ns: () -> Integer
+    def to_ns: (Float seconds) -> Integer
+  end
+end

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification
+name: supervisor_machines
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Abdelkader Boudih
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: chrono_machines
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.11'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.11'
+- !ruby/object:Gem::Dependency
+  name: deadline_machines
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: rb_sys
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+description: 'A supervision tree that restarts what died the way OTP would: one_for_one,
+  one_for_all or rest_for_one, bounded by a restart intensity that escalates instead
+  of thrashing. The decisions come from the no_std supervisor-machines crate; Ruby
+  owns the threads, the restart backoff comes from chrono_machines and the shutdown
+  budget from deadline_machines.'
+email:
+- oss@seuros.com
+executables: []
+extensions:
+- ext/supervisor_machines_native/extconf.rb
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Cargo.lock
+- Cargo.toml
+- LICENSE.txt
+- README.md
+- ext/supervisor_machines_native/core/Cargo.toml
+- ext/supervisor_machines_native/core/src/lib.rs
+- ext/supervisor_machines_native/core/src/supervisor.rs
+- ext/supervisor_machines_native/extconf.rb
+- ext/supervisor_machines_native/ffi/Cargo.toml
+- ext/supervisor_machines_native/ffi/extconf.rb
+- ext/supervisor_machines_native/ffi/src/decider.rs
+- ext/supervisor_machines_native/ffi/src/lib.rs
+- lib/supervisor_machines.rb
+- lib/supervisor_machines/errors.rb
+- lib/supervisor_machines/supervisor.rb
+- lib/supervisor_machines/version.rb
+- sig/supervisor_machines.rbs
+homepage: https://github.com/seuros/supervisor_machines
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/seuros/supervisor_machines
+  changelog_uri: https://github.com/seuros/supervisor_machines/blob/main/CHANGELOG.md
+  bug_tracker_uri: https://github.com/seuros/supervisor_machines/issues
+  rubygems_mfa_required: 'true'
+  cargo_crate_name: supervisor_machines_native
+  cargo_manifest_path: ext/supervisor_machines_native/ffi/Cargo.toml
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 4.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.20
+specification_version: 4
+summary: OTP-style supervision trees over Ruby threads
+test_files: []