service_skeleton 0.0.0.1.ENOTAG → 0.0.0.2.g46c1e0e

data/lib/service_skeleton/signal_manager.rb

@@ -1,202 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "./logging_helpers"
-
-module ServiceSkeleton
-  # Manage signals in a sane and safe manner.
-  #
-  # Signal handling is a shit of a thing.  The code that runs when a signal is
-  # triggered can't use mutexes (which are used in all sorts of places you
-  # might not expect, like Logger!) or anything else that might block.  This
-  # greatly constrains what you can do inside a signal handler, so the standard
-  # approach is to stuff a character down a pipe, and then have the *real*
-  # signal handling run later.
-  #
-  # Also, there's always the (slim) possibility that something else might have
-  # hooked into a signal we want to receive.  Because only a single signal
-  # handler can be active for a given signal at a time, we need to "chain" the
-  # existing handler, by calling the previous signal handler from our signal
-  # handler after we've done what we need to do.  This class takes care of
-  # that, too, because it's a legend.
-  #
-  # So that's what this class does: it allows you to specify signals and
-  # associated blocks of code to run, it sets up signal handlers which send
-  # notifications to a background thread and chain correctly, and it manages
-  # the background thread to receive the notifications and execute the
-  # associated blocks of code outside of the context of the signal handler.
-  #
-  class SignalManager
-    include ServiceSkeleton::LoggingHelpers
-
-    # Setup a signal handler instance.
-    #
-    # @param logger [Logger] the logger to use for all the interesting information
-    #   about what we're up to.
-    #
-    def initialize(logger:, counter:, signals:)
-      @logger, @signal_counter, @signal_list = logger, counter, signals
-
-      @registry = Hash.new { |h, k| h[k] = SignalHandler.new(k) }
-
-      @signal_list.each do |sig, proc|
-        @registry[signum(sig)] << proc
-      end
-    end
-
-    def run
-      logger.info(logloc) { "Starting signal manager for #{@signal_list.length} signals" }
-
-      @r, @w = IO.pipe
-
-      install_signal_handlers
-
-      signals_loop
-    ensure
-      remove_signal_handlers
-    end
-
-    def shutdown
-      @r.close
-    end
-
-    private
-
-    attr_reader :logger
-
-    def signals_loop
-      #:nocov:
-      loop do
-        begin
-          if ios = IO.select([@r])
-            if ios.first.include?(@r)
-              if ios.first.first.eof?
-                logger.info(logloc) { "Signal pipe closed; shutting down" }
-                break
-              else
-                c = ios.first.first.read_nonblock(1)
-                logger.debug(logloc) { "Received character #{c.inspect} from signal pipe" }
-                handle_signal(c)
-              end
-            else
-              logger.error(logloc) { "Mysterious return from select: #{ios.inspect}" }
-            end
-          end
-        rescue IOError
-          # Something has gone terribly wrong here... bail
-          break
-        rescue StandardError => ex
-          log_exception(ex) { "Exception in select loop" }
-        end
-      end
-      #:nocov:
-    end
-
-    # Given a character (presumably) received via the signal pipe, execute the
-    # associated handler.
-    #
-    # @param char [String] a single character, corresponding to an entry in the
-    #   signal registry.
-    #
-    # @return [void]
-    #
-    def handle_signal(char)
-      if @registry.has_key?(char.ord)
-        handler = @registry[char.ord]
-        logger.debug(logloc) { "#{handler.signame} received" }
-        @signal_counter.increment(labels: { signal: handler.signame.to_s })
-
-        begin
-          handler.call
-        rescue StandardError => ex
-          log_exception(ex) { "Exception while calling signal handler" }
-        end
-      else
-        logger.error(logloc) { "Unrecognised signal character: #{char.inspect}" }
-      end
-    end
-
-    def install_signal_handlers
-      @registry.values.each do |h|
-        h.write_pipe = @w
-        h.hook
-      end
-    end
-
-    def signum(spec)
-      if spec.is_a?(Integer)
-        return spec
-      end
-
-      if spec.is_a?(Symbol)
-        str = spec.to_s
-      elsif spec.is_a?(String)
-        str = spec.dup
-      else
-        raise ArgumentError,
-              "Unsupported class (#{spec.class}) of signal specifier #{spec.inspect}"
-      end
-
-      str.sub!(/\ASIG/i, '')
-
-      if Signal.list[str.upcase]
-        Signal.list[str.upcase]
-      else
-        raise ArgumentError,
-              "Unrecognised signal specifier #{spec.inspect}"
-      end
-    end
-
-    def remove_signal_handlers
-      @registry.values.each { |h| h.unhook }
-    end
-
-    class SignalHandler
-      attr_reader :signame
-      attr_writer :write_pipe
-
-      def initialize(signum)
-        @signum = signum
-        @callbacks = []
-
-        @signame = Signal.list.invert[@signum]
-      end
-
-      def <<(proc)
-        @callbacks << proc
-      end
-
-      def call
-        @callbacks.each { |cb| cb.call }
-      end
-
-      def hook
-        @handler = ->(_) do
-          #:nocov:
-          @write_pipe.write_nonblock(@signum.chr) rescue nil
-          @chain.call if @chain.respond_to?(:call)
-          #:nocov:
-        end
-
-        @chain = Signal.trap(@signum, &@handler)
-      end
-
-      def unhook
-        #:nocov:
-        tmp_handler = Signal.trap(@signum, "IGNORE")
-        if tmp_handler == @handler
-          # The current handler is ours, so we can replace it
-          # with the chained handler
-          Signal.trap(@signum, @chain)
-        else
-          # The current handler *isn't* ours, so we better
-          # put it back, because whoever owns it might get
-          # angry.
-          Signal.trap(@signum, tmp_handler)
-        end
-        #:nocov:
-      end
-    end
-
-    private_constant :SignalHandler
-  end
-end

data/lib/service_skeleton/signals_methods.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module ServiceSkeleton
-  module SignalsMethods
-    def registered_signal_handlers
-      @registered_signal_handlers || []
-    end
-
-    def hook_signal(sigspec, &blk)
-      @registered_signal_handlers ||= []
-
-      @registered_signal_handlers << [sigspec, blk]
-    end
-  end
-end

data/lib/service_skeleton/ultravisor_children.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-module ServiceSkeleton
-  module UltravisorChildren
-    def register_ultravisor_children(ultravisor, config:, metrics_registry:)
-      begin
-        ultravisor.add_child(
-          id: self.service_name.to_sym,
-          klass: self,
-          method: :run,
-          args: [config: config, metrics: metrics_registry],
-          access: :unsafe
-        )
-      rescue Ultravisor::InvalidKAMError
-        raise ServiceSkeleton::Error::InvalidServiceClassError,
-              "Class #{self.to_s} does not implement the `run' instance method"
-      end
-    end
-  end
-end

data/lib/service_skeleton/ultravisor_loggerstash.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-module ServiceSkeleton
-  module UltravisorLoggerstash
-    def logstash_writer
-      #:nocov:
-      @ultravisor[:logstash_writer].unsafe_instance
-      #:nocov:
-    end
-  end
-end

data/ultravisor/.yardopts

@@ -1 +0,0 @@
---markup markdown

data/ultravisor/Guardfile

@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-guard 'rspec',
-      cmd: "bundle exec rspec",
-      all_on_start: true,
-      all_after_pass: true do
-  watch(%r{^spec/.+_spec\.rb$})
-  watch(%r{^spec/.+_methods\.rb$})
-  watch(%r{^lib/}) { "spec" }
-end

data/ultravisor/README.md

@@ -1,404 +0,0 @@
-> # WARNING WARNING WARNING
->
-> This README is, at least in part, speculative fiction.  I practice
-> README-driven development, and as such, not everything described in here
-> actually exists yet, and what does exist may not work right.
-
-Ultravisor is like a supervisor, but... *ULTRA*.  The idea is that you specify
-objects to instantiate and run in threads, and then the Ultravisor makes that
-happen behind the scenes, including logging failures, restarting if necessary,
-and so on.  If you're familiar with Erlang supervision trees, then Ultravisor
-will feel familiar to you, because I stole pretty much every good idea that
-is in Ultravisor from Erlang.  You will get a lot of very excellent insight
-from reading [the Erlang/OTP Supervision Principles](http://erlang.org/doc/design_principles/sup_princ.html).
-
-# Usage
-
-This section gives you a basic overview of the high points of how Ultravisor
-can be used.  It is not intended to be an exhaustive reference of all possible
-options; the {Ultravisor} class API documentation provides every possible option
-and its meaning.
-
-
-## The Basics
-
-Start by loading the code:
-
-    require "ultravisor"
-
-Creating a new Ultravisor is a matter of instantiating a new object:
-
-    u = Ultravisor.new
-
-In order for it to be useful, though, you'll need to add one or more children
-to the Ultravisor instance, which can either be done as part of the call to
-`.new`, or afterwards, as you see fit:
-
-    # Defining a child in the constructor
-    u = Ultravisor.new(children: [{id: :child, klass: Child, method: :run}])
-
-    # OR define it afterwards
-    u = Ultravisor.new
-    u.add_child(id: :my_child, klass: Child, method: :run)
-
-Once you have an Ultravisor with children configured, you can set it running:
-
-    u.run
-
-This will block until the Ultravisor terminates, one way or another.
-
-We'll learn about other available initialization arguments, and all the other
-features of Ultravisor, in the following sections.
-
-
-## Defining Children
-
-As children are the primary reason Ultravisor exists, it is worth getting a handle
-on them first.
-
-Defining children, as we saw in the introduction, can be done by calling
-{Ultravisor#add_child} for each child you want to add, or else you can provide
-a list of children to start as part of the {Ultravisor.new} call, using the
-`children` named argument.  You can also combine the two approaches, if some
-children are defined statically, while others only get added conditionally.
-
-Let's take another look at that {Ultravisor#add_child} method from earlier:
-
-    u.add_child(id: :my_child, klass: Child, method: :run)
-
-First up, every child has an ID.  This is fairly straightforward -- it's a
-unique ID (within a given Ultravisor) that refers to the child.  Attempting to
-add two children with the same ID will raise an exception.
-
-The `class` and `method` arguments require a little more explanation.  One
-of the foundational principles of "fail fast" is "clean restart" -- that is, if you
-do need to restart something, it's important to start with as clean a state as possible.
-Thus, if a child needs to be restarted, we don't want to reuse an existing object, which
-may be in a messy and unuseable state.  Instead, we want a clean, fresh object to work on.
-That's why you specify a `class` when you define a child -- it is a new instance of that
-class that will be used every time the child is started (or restarted).
-
-The `method` argument might now be obvious.  Once the new instance of the
-specified `class` exists, the Ultravisor will call the specified `method` to start
-work happening.  It is expected that this method will ***not return***, in most cases.
-So you probably want some sort of infinite loop.
-
-You might think that this is extremely inflexible, only being able to specify a class
-and a method to call.  What if you want to pass in some parameters?  Don't worry, we've
-got you covered:
-
-    u.add_child(
-      id: :my_child,
-      klass: Child,
-      args: ['foo', 42, x: 1, y: 2],
-      method: :run,
-    )
-
-The call to `Child.new` can take arbitrary arguments, just by defining an array
-for the `args` named parameter.  Did you know you can define a hash inside an
-array like `['foo', 'bar', x: 1, y: 2] => ['foo', 'bar', {:x => 1, :y => 2}]`?
-I didn't, either, until I started working on Ultravisor, but you can, and it
-works *exactly* like named parameters in method calls.
-
-You can also add children after the Ultravisor has been set running:
-
-    u = Ultravisor.new
-
-    u.add_child(id: :c1, klass: SomeWorker, method: :run)
-
-    u.run   # => starts running an instance of SomeWorker, doesn't return
-
-    # In another thread...
-    u.add_child(id: :c2, klass: OtherWorker, method: go!)
-
-    # An instance of OtherWorker will be created and set running
-
-If you add a child to an already-running Ultravisor, that child will immediately be
-started running, almost like magic.
-
-
-### Ordering of Children
-
-The order in which children are defined is important.  When children are (re)started,
-they are always started in the order they were defined.  When children are stopped,
-either because the Ultravisor is shutting down, or because of a [supervision
-strategy](#supervision-strategies), they are always stopped in the *reverse* order
-of their definition.
-
-All child specifications passed to {Ultravisor.new} always come first, in the
-order they were in the array.  Any children defined via calls to
-{Ultravisor#add_child} will go next, in the order the `add_child` calls were
-made.
-
-
-## Restarting Children
-
-One of the fundamental purposes of a supervisor like Ultravisor is that it restarts
-children if they crash, on the principle of "fail fast".  There's no point failing fast
-if things don't get automatically fixed.  This is the default behaviour of all
-Ultravisor children.
-
-Controlling how children are restarted is the purpose of the "restart policy",
-which is controlled by the `restart` and `restart_policy` named arguments in
-the child specification.  For example, if you want to create a child that will
-only ever be run once, regardless of what happens to it, then use `restart:
-:never`:
-
-    u.add_child(
-      id: :my_one_shot_child,
-      klass: Child,
-      method: :run_maybe,
-      restart: :never
-    )
-
-If you want a child which gets restarted if its `method` raises an exception,
-but *not* if it runs to completion without error, then use `restart: :on_failure`:
-
-    u.add_child(
-      id: :my_run_once_child,
-      klass: Child,
-      method: :run_once,
-      restart: :on_failure
-    )
-
-### The Limits of Failure
-
-While restarting is great in general, you don't particularly want to fill your
-logs with an endlessly restarting child -- say, because it doesn't have
-permission to access a database.  To solve that problem, an Ultravisor will
-only attempt to restart a child a certain number of times before giving up and
-exiting itself.  The parameters of how this works are controlled by the
-`restart_policy`, which is itself a hash:
-
-    u.add_child(
-      id: :my_restartable_child,
-      klass: Child,
-      method: :run,
-      restart_policy: {
-        period: 5,
-        retries: 2,
-        delay: 1,
-      }
-    )
-
-The meaning of each of the `restart_policy` keys is best explained as part
-of how Ultravisor restarts children.
-
-When a child needs to be restarted, Ultravisor first waits a little while
-before attempting the restart.  The amount of time to wait is specified
-by the `delay` value in the `restart_policy`.  Then a new instance of the
-`class` is instantiated, and the `method` is called on that instance.
-
-The `period` and `retries` values of the `restart_policy` come into play
-when the child exits repeatedly.  If a single child needs to be restarted
-more than `retries` times in `period` seconds, then instead of trying to
-restart again, Ultravisor gives up.  It doesn't try to start the child
-again, it terminates all the *other* children of the Ultravisor, and
-then it exits.  Note that the `delay` between restarts is *not* part
-of the `period`; only time spent actually running the child is
-accounted for.
-
-
-## Managed Child Termination
-
-If children need to be terminated, by default, child threads are simply
-forcibly terminated by calling {Thread#kill} on them. However, for workers
-which hold resources, this can cause problems.
-
-Thus, it is possible to control both how a child is terminated, and how long
-to wait for that termination to occur, by using the `shutdown` named argument
-when you add a child (either via {Ultravisor#add_child}, or as part of the
-`children` named argument to {Ultravisor.new}), like this:
-
-    u.add_child(
-      id: :fancy_worker,
-      shutdown: {
-        method: :gentle_landing,
-        timeout: 30
-      }
-    )
-
-When a child with a custom shutdown policy needs to be terminated, the
-method named in the `method` key is called on the instance of `class` that
-represents that child.  Once the shutdown has been signalled to the
-worker, up to `timeout` seconds is allowed to elapse.  If the child thread has
-not terminated by this time, the thread is forcibly terminated by calling
-{Thread#kill}.  This timeout prevents shutdown or group restart from hanging
-indefinitely.
-
-Note that the `method` specified in the `shutdown` specification should
-signal the worker to terminate, and then return immediately.  It should
-*not* wait for termination itself.
-
-
-## Supervision Strategies
-
-When a child needs to be restarted, by default only the child that exited
-will be restarted.  However, it is possible to cause other
-children to be restarted as well, if that is necessary.  To do that, you
-use the `strategy` named parameter when creating the Ultravisor:
-
-    u = Ultravisor.new(strategy: :one_for_all)
-
-The possible values for the strategy are:
-
-* `:one_for_one` -- the default restart strategy, this simply causes the
-  child which exited to be started again, in line with its restart policy.
-
-* `:all_for_one` -- if any child needs to be restarted, all children of the
-  Ultravisor get terminated in reverse of their start order, and then all
-  children are started again, except those which are `restart: :never`, or
-  `restart: :on_failure` which had not already exited without error.
-
-* `:rest_for_one` -- if any child needs to be restarted, all children of
-  the Ultravisor which are *after* the restarted child get terminated
-  in reverse of their start order, and then all children are started again,
-  except those which are `restart: :never`, or `restart: :on_failure` which
-  had not already exited without error.
-
-
-## Interacting With Child Objects
-
-Since the Ultravisor is creating the object instances that run in the worker
-threads, you don't automatically have access to the object instance itself.
-This is somewhat by design -- concurrency bugs are hell.  However, there *are*
-ways around this, if you need to.
-
-
-### The power of cast / call
-
-A common approach for interacting with an object in an otherwise concurrent
-environment is the `cast` / `call` pattern.  From the outside, the interface
-is quite straightforward:
-
-```
-u = Ultravisor.new(children: [
-      { id: :castcall, klass: CastCall, method: :run, enable_castcall: true }
-    ])
-
-# This will return `nil` immediately
-u[:castcall].cast.some_method
-
-# This will, at some point in the future, return whatever `CastCall#to_s` could
-u[:castcall].call.some_method
-```
-
-To enable `cast` / `call` support for a child, you must set the `enable_castcall`
-keyword argument on the child.  This is because failing to process `cast`s and
-`call`s can cause all sorts of unpleasant backlogs, so children who intend to
-receive (and process) `cast`s and `call`s must explicitly opt-in.
-
-The interface to the object from outside is straightforward.  You get a
-reference to the instance of {Ultravisor::Child} for the child you want to talk
-to (which is returned by {Ultravisor#add_child}, or {Ultravisor#[]}), and then
-call `child.cast.<method>` or `child.call.<method>`, passing in arguments as
-per normal.  Any public method can be the target of the `cast` or `call`, and you
-can pass in any arguments you like, *including blocks* (although bear in mind that
-any blocks passed will be run in the child instance's thread, and many
-concurrency dragons await the unwary).
-
-The difference between the `cast` and `call` methods is in whether or not a
-return value is expected, and hence when the method call chained through
-`cast` or `call` returns.
-
-When you call `cast`, the real method call gets queued for later execution,
-and since no return value is expected, the `child.cast.<method>` returns
-`nil` immediately and your code gets on with its day.  This is useful
-when you want to tell the worker something, or instruct it to do something,
-but there's no value coming back.
-
-In comparison, when you call `call`, the real method call still gets queued,
-but the calling code blocks, waiting for the return value from the queued
-method call.  This may seem pointless -- why have concurrency that blocks? --
-but the value comes from the synchronisation.  The method call only happens
-when the worker loop calls `process_castcall`, which it can do at a time that
-suits it, and when it knows that nothing else is going on that could cause
-problems.
-
-One thing to be aware of when interacting with a worker instance is that it may
-crash, and be restarted by the Ultravisor, before it gets around to processing
-a queued message.  If you used `child.cast`, then the method call is just...
-lost, forever.  On the other hand, if you used `child.call`, then an
-{Ultravisor::ChildRestartedError} exception will be raised, which you can deal
-with as you see fit.
-
-The really interesting part is what happens *inside* the child instance.  The
-actual execution of code in response to the method calls passed through `cast`
-and `call` will only happen when the running instance of the child's class
-calls `process_castcall`.  When that happens, all pending casts and calls will
-be executed.  Since this happens within the same thread as the rest of the
-child instance's code, it's a lot safer than trying to synchronise everything
-with locks.
-
-You can, of course, just call `process_castcall` repeatedly, however that's a
-somewhat herp-a-derp way of doing it.  The `castcall_fd` method in the running
-instance will return an IO object which will become readable whenever there is
-a pending `cast` or `call` to process.  Thus, if you're using `IO.select` or
-similar to wait for work to do, you can add `castcall_fd` to the readable set
-and only call `process_castcall` when the relevant IO object comes back.  Don't
-actually try *reading* from it yourself; `process_castcall` takes care of all that.
-
-If you happen to have a child class whose *only* purpose is to process `cast`s
-and `call`s, you should configure the Ultravisor to use `process_castcall_loop`
-as its entry method.  This is a wrapper method which blocks on `castcall_fd`
-becoming readable, and loops infinitely.
-
-It is important to remember that not all concurrency bugs can be prevented by
-using `cast` / `call`.  For example, read-modify-write operations will still
-cause all the same problems they always do, so if you find yourself calling
-`child.call`, modifying the value returned, and then calling `child.cast`
-with that modified value, you're in for a bad time.
-
-
-### Direct (Unsafe) Instance Access
-
-If you have a worker class which you're *really* sure is safe against concurrent
-access, you can eschew the convenience and safety of `cast` / `call`, and instead
-allow direct access to the worker instance object.
-
-To do this, specify `access: :unsafe` in the child specification, and then
-call `child.unsafe_instance` to get the instance object currently in play.
-
-Yes, the multiple mentions of `unsafe` are there deliberately, and no, I won't
-be removing them.  They're there to remind you, always, that what you're doing
-is unsafe.
-
-If the child is restarting at the time `child.unsafe_instance` is called,
-the call will block until the child worker is started again, after which
-you'll get the newly created worker instance object.  The worker could crash
-again at any time, of course, leaving you with a now out-of-date object
-that is no longer being actively run.  It's up to you to figure out how to
-deal with that.  If the Ultravisor associated with the child
-has terminated, your call to `child.unsafe_instance` will raise an
-{Ultravisor::ChildRestartedError}.
-
-Why yes, Gracie, there *are* a lot of things that can go wrong when using
-direct instance object access.  Still wondering why those `unsafe`s are in
-the name?
-
-
-## Supervision Trees
-
-Whilst a collection of workers is a neat thing to have, more powerful systems
-can be constructed if supervisors can, themselves, be supervised.  Primarily
-this is useful when recovering from persistent errors, because you can use
-a higher-level supervisor to restart an entire tree of workers which has one
-which is having problems.
-
-Creating a supervision tree is straightforward.  Because Ultravisor works by
-instantiating plain old ruby objects, and Ultravisor is, itself, a plain old
-ruby class, you use it more-or-less like you would any other object:
-
-    u = Ultravisor.new
-    u.add_child(id: :sub_sup, klass: Ultravisor, method: :run, args: [children: [...]])
-
-That's all there is to it.  Whenever the parent Ultravisor wants to work on the
-child Ultravisor, it treats it like any other child, asking it to terminate,
-start, etc, and the child Ultravisor's work consists of terminating, starting,
-etc all of its children.
-
-The only difference in default behaviour between a regular worker child and an
-Ultravisor child is that an Ultravisor's `shutdown` policy is automatically set
-to `method: :stop!, timeout: :infinity`.  This is because it is *very* bad news
-to forcibly terminate an Ultravisor before its children have stopped -- all
-those children just get cast into the VM, never to be heard from again.

data/ultravisor/lib/ultravisor/child/call.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-class Ultravisor::Child::Call
-  attr_reader :method_name
-
-  def initialize(method_name, args, blk, rv_q, rv_fail)
-    @method_name, @args, @blk, @rv_q, @rv_fail = method_name, args, blk, rv_q, rv_fail
-  end
-
-  def go!(receiver)
-    @rv_q << receiver.__send__(@method_name, *@args, &@blk)
-  rescue Exception => ex
-    @rv_q << @rv_fail
-    raise
-  ensure
-    @rv_q.close
-  end
-
-  def child_restarted!
-    @rv_q << @rv_fail
-  end
-end