glazed 0.1.0

data/lib/glazed/dialects.rb

@@ -0,0 +1,65 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'glazed'
+require 'glazed/finder'
+require 'glazed/must'
+require 'glazed/dialects/base'
+
+module Glazed
+  # Namespace of dialects, looked up by name.
+  module Dialects
+    #: type resolver = ^(String name) -> singleton(Base)?
+
+    @resolvers = [] #: Array[Dialects::resolver]
+    @resolvers_lock = Mutex.new #: Thread::Mutex
+
+    # Finds a dialect class by snake_case name, then asks each resolver in
+    # registration order.
+    #
+    # @param name [String, Symbol] dialect name, such as "ios"
+    # @return [Class<Base>] the dialect class
+    # @raise [Finder::NotFound] when there is none
+    # @raise [Must::Violation] when the name resolves to something other than a dialect
+    #: (String | Symbol name) -> singleton(Base)
+    def self.find(name)
+      found = Finder.maybe_find(self, 'glazed/dialects', name) || resolve(name.to_s)
+      raise Finder::NotFound, "Couldn't find #{name.inspect} for #{self}" unless found
+      return found if found.is_a?(Class) && found < Base
+
+      raise Must::Violation, "#{found} is not a dialect"
+    end
+
+    # Registers a resolver for names that no dialect class or file answers to,
+    # such as dialects generated from another format.
+    #
+    # @yieldparam name [String] dialect name
+    # @yieldreturn [Class<Base>, nil] the dialect, or nil when the resolver has none by that name
+    # @return [void]
+    # @example
+    #   Glazed::Dialects.add_resolver { |name| GENERATED[name] }
+    #: () { (String name) -> singleton(Base)? } -> void
+    def self.add_resolver(&resolver)
+      @resolvers_lock.synchronize { @resolvers << resolver }
+    end
+
+    # Removes a resolver registered with {add_resolver}.
+    #
+    # @param resolver [Proc] the resolver
+    # @return [void]
+    #: (Dialects::resolver resolver) -> void
+    def self.remove_resolver(resolver)
+      @resolvers_lock.synchronize { @resolvers.delete(resolver) }
+    end
+
+    # @param name [String] dialect name
+    # @return [Class<Base>, nil] the first dialect a resolver returns
+    #: (String name) -> singleton(Base)?
+    private_class_method def self.resolve(name)
+      resolvers = @resolvers_lock.synchronize { @resolvers.dup }
+      resolvers.lazy.filter_map { it.call(name) }.first
+    end
+  end
+end

data/lib/glazed/finder.rb

@@ -0,0 +1,62 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+module Glazed
+  # Resolves plugin classes by name, requiring `<prefix>/<name>` on demand so
+  # other gems can provide implementations.
+  module Finder
+    # Raised when no implementation is found for a name.
+    class NotFound < Glazed::Error; end
+
+    # Looks up the constant for a snake_case name under a namespace.
+    #
+    # @param namespace [Module] namespace holding implementations
+    # @param prefix [String] require path prefix for the namespace
+    # @param name [String, Symbol] snake_case implementation name
+    # @return [Module] the implementation
+    # @raise [NotFound] when the constant cannot be found even after requiring
+    # @example
+    #   Glazed::Finder.find(Glazed::Storages, 'glazed/storages', 's3')
+    #: (Module[untyped] namespace, String prefix, String | Symbol name) -> Module[untyped]
+    def self.find(namespace, prefix, name)
+      maybe_find(namespace, prefix, name) or raise NotFound, "Couldn't find #{name.inspect} for #{namespace}"
+    end
+
+    # Looks up the constant for a snake_case name, returning nil when absent.
+    #
+    # @param namespace [Module] namespace holding implementations
+    # @param prefix [String] require path prefix for the namespace
+    # @param name [String, Symbol] snake_case implementation name
+    # @return [Module, nil] the implementation
+    # @raise [LoadError] when the implementation exists but fails to load its dependencies
+    #: (Module[untyped] namespace, String prefix, String | Symbol name) -> Module[untyped]?
+    def self.maybe_find(namespace, prefix, name)
+      constant_name = name.to_s.split('_').map(&:capitalize).join
+      return lookup(namespace, constant_name) if namespace.const_defined?(constant_name, false)
+
+      path = "#{prefix}/#{name}"
+      begin
+        require path
+      rescue LoadError => e
+        raise unless e.path == path
+
+        return nil
+      end
+      lookup(namespace, constant_name) if namespace.const_defined?(constant_name, false)
+    end
+
+    # @param namespace [Module] namespace holding implementations
+    # @param constant_name [String] constant to read
+    # @return [Module] the constant, which must be a module or class
+    # @raise [NotFound] when the constant is not a module
+    #: (Module[untyped] namespace, String constant_name) -> Module[untyped]
+    private_class_method def self.lookup(namespace, constant_name)
+      value = namespace.const_get(constant_name, false)
+      raise NotFound, "#{namespace}::#{constant_name} is not a module" unless value.is_a?(Module)
+
+      value
+    end
+  end
+end

data/lib/glazed/must.rb

@@ -0,0 +1,54 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+module Glazed
+  # Runtime counterpart of Sorbet's T.must for a value the type checker cannot
+  # prove present. The reason names the invariant that makes it so.
+  module Must
+    # Raised when an asserted invariant does not hold.
+    class Violation < StandardError; end
+
+    # Returns the value, or raises with the reason when it is nil.
+    #
+    # @param value [Object, nil] value expected to be present
+    # @yieldreturn [String] the invariant that makes the value present
+    # @return [Object] the value
+    # @raise [Violation] when the value is nil
+    #: [T] (T? value) { () -> String } -> T
+    def self.because(value, &_block)
+      case value
+      when nil then raise Violation, yield
+      else value
+      end
+    end
+
+    # Returns the value, or raises when it is nil.
+    #
+    # @param value [Object, nil] value expected to be present
+    # @return [Object] the value
+    # @raise [Violation] when the value is nil
+    #: [T] (T? value) -> T
+    def self.expect(value)
+      case value
+      when nil then raise Violation, 'expected a value, got nil'
+      else value
+      end
+    end
+
+    # Returns the value narrowed to the class or module, or raises.
+    #
+    # @param value [Object] value to narrow
+    # @param klass [Module] expected class or module
+    # @return [Object] the value
+    # @raise [Violation] when the value is not a klass
+    #: [T] (untyped value, Module[T] klass) -> T
+    def self.be_a(value, klass)
+      case value
+      when klass then value
+      else raise Violation, "expected #{klass}, got #{value.class}"
+      end
+    end
+  end
+end

data/lib/glazed/orchestrator/policy.rb

@@ -0,0 +1,184 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'google/protobuf'
+require 'glazed'
+require 'glazed/proto'
+
+module Glazed
+  class Orchestrator
+    # How node schedules change on inventory changes, signals, claims and
+    # completed runs. Each function returns a new record and leaves the given
+    # one untouched, so state backends may call them more than once.
+    module Policy
+      # Seconds between runs when the schedule sets none.
+      DEFAULT_INTERVAL = 3600 #: Integer
+      # Seconds a signal waits for further signals when the debounce sets none.
+      DEFAULT_DELAY = 60 #: Integer
+      # Seconds after the first signal by which a node runs when the debounce sets none.
+      DEFAULT_MAX_WAIT = 600 #: Integer
+
+      # Record of a node new to the inventory, due within the schedule's jitter.
+      #
+      # @param workflow [String] workflow name
+      # @param node [String] node name
+      # @param now [Time] current time
+      # @param schedule [Proto::V1::Schedule, nil] the schedule
+      # @param random [Random] source of jitter
+      # @return [Proto::V1::NodeSchedule] the record
+      #: (String workflow, String node, now: Time, schedule: Proto::V1::Schedule?, random: Random) -> Proto::V1::NodeSchedule
+      def self.discovered(workflow, node, now:, schedule:, random:)
+        due = timestamp(now + jitter(schedule, random))
+        Proto::V1::NodeSchedule.new(workflow:, node:, due_at: due, scheduled_at: due, due_reason: 'inventory')
+      end
+
+      # Applies a signal with trailing debounce: the node becomes due the
+      # debounce delay after the latest signal, but no later than the maximum
+      # wait after the first signal not yet followed by a run. An earlier
+      # scheduled run is kept, unless the node is running, as that run started
+      # before the signal.
+      #
+      # @param record [Proto::V1::NodeSchedule, nil] the node's record; nil when it has none yet
+      # @param signal [Proto::V1::Signal] the signal
+      # @param workflow [String] workflow name
+      # @param now [Time] current time, used when the signal has no reception time
+      # @param debounce [Proto::V1::Debounce, nil] the debounce settings
+      # @return [Proto::V1::NodeSchedule] the record
+      #: (Proto::V1::NodeSchedule? record, Proto::V1::Signal signal, workflow: String, now: Time, debounce: Proto::V1::Debounce?) -> Proto::V1::NodeSchedule
+      def self.signal(record, signal, workflow:, now:, debounce:)
+        at = time(signal.received_at) || now
+        updated = record&.dup || Proto::V1::NodeSchedule.new(workflow:, node: signal.node)
+        running = running?(updated, now)
+        window = time(updated.signal_window_started_at)
+        started = time(updated.last_started_at)
+        window = at if window.nil? || (running && started && window <= started)
+
+        due = if signal.immediate
+          at
+        else
+          [at + nonzero(debounce&.delay_seconds, DEFAULT_DELAY), window + nonzero(debounce&.max_wait_seconds, DEFAULT_MAX_WAIT)].min
+        end
+        scheduled = time(updated.scheduled_at)
+        if !running && scheduled && scheduled <= due
+          updated.due_at = timestamp(scheduled)
+        else
+          updated.due_at = timestamp(due)
+          updated.due_reason = signal.source
+        end
+        updated.signal_window_started_at = timestamp(window)
+        updated
+      end
+
+      # Leases a due node to an orchestrator.
+      #
+      # @param record [Proto::V1::NodeSchedule, nil] the node's record
+      # @param owner [String] identifier of the claiming orchestrator and batch
+      # @param now [Time] current time
+      # @param lease_seconds [Integer] seconds the lease lasts
+      # @return [Proto::V1::NodeSchedule, nil] the leased record; the record unchanged when it is not due or leased to another
+      #: (Proto::V1::NodeSchedule? record, owner: String, now: Time, lease_seconds: Integer) -> Proto::V1::NodeSchedule?
+      def self.claim(record, owner:, now:, lease_seconds:)
+        return record unless record
+        return record if running?(record, now)
+
+        due = time(record.due_at)
+        return record unless due && due <= now
+
+        record.dup.tap do |claimed|
+          claimed.lease_owner = owner
+          claimed.lease_until = timestamp(now + lease_seconds)
+          claimed.last_started_at = timestamp(now)
+        end
+      end
+
+      # Records the outcome of a run and schedules the next one. A signal that
+      # arrived during the run keeps its due date when that is earlier.
+      #
+      # @param record [Proto::V1::NodeSchedule, nil] the node's record
+      # @param owner [String] identifier the run claimed the node with
+      # @param finished_at [Time] when the run finished
+      # @param error [String, nil] why the run failed; nil when it succeeded
+      # @param schedule [Proto::V1::Schedule, nil] the schedule
+      # @param random [Random] source of jitter
+      # @return [Proto::V1::NodeSchedule, nil] the record; unchanged when another orchestrator holds it
+      #: (Proto::V1::NodeSchedule? record, owner: String, finished_at: Time, error: String?, schedule: Proto::V1::Schedule?, random: Random) -> Proto::V1::NodeSchedule?
+      def self.completed(record, owner:, finished_at:, error:, schedule:, random:)
+        return record unless record && record.lease_owner == owner
+
+        interval = nonzero(schedule&.interval_seconds, DEFAULT_INTERVAL)
+        interval = nonzero(schedule&.retry_interval_seconds, interval) if error
+        due = finished_at + interval + jitter(schedule, random)
+        window = time(record.signal_window_started_at)
+        started = time(record.last_started_at)
+        pending = window && started && window > started
+
+        record.dup.tap do |updated|
+          updated.lease_owner = ''
+          updated.lease_until = nil
+          updated.last_finished_at = timestamp(finished_at)
+          updated.last_failed = !error.nil?
+          updated.last_error = error || ''
+          updated.consecutive_failures = error ? record.consecutive_failures + 1 : 0
+          updated.scheduled_at = timestamp(due)
+          current = time(record.due_at)
+          unless pending && current && current <= due
+            updated.due_at = timestamp(due)
+            updated.due_reason = error ? 'retry' : 'schedule'
+          end
+          updated.signal_window_started_at = nil unless pending
+        end
+      end
+
+      # Gives up a lease without recording a run, such as when the orchestrator is interrupted.
+      #
+      # @param record [Proto::V1::NodeSchedule, nil] the node's record
+      # @param owner [String] identifier the node was claimed with
+      # @return [Proto::V1::NodeSchedule, nil] the record without the lease; unchanged when another orchestrator holds it
+      #: (Proto::V1::NodeSchedule? record, owner: String) -> Proto::V1::NodeSchedule?
+      def self.released(record, owner:)
+        return record unless record && record.lease_owner == owner
+
+        record.dup.tap do |updated|
+          updated.lease_owner = ''
+          updated.lease_until = nil
+        end
+      end
+
+      # @param record [Proto::V1::NodeSchedule] the node's record
+      # @param now [Time] current time
+      # @return [Boolean] whether an orchestrator holds an unexpired lease on the node
+      #: (Proto::V1::NodeSchedule record, Time now) -> bool
+      def self.running?(record, now)
+        lease_until = time(record.lease_until)
+        !record.lease_owner.empty? && !lease_until.nil? && lease_until > now
+      end
+
+      # @param timestamp [Google::Protobuf::Timestamp, nil] the timestamp
+      # @return [Time, nil] the time, or nil when unset
+      #: (Google::Protobuf::Timestamp? timestamp) -> Time?
+      def self.time(timestamp) = timestamp&.to_time
+
+      # @param time [Time] the time
+      # @return [Google::Protobuf::Timestamp] the timestamp
+      #: (Time time) -> Google::Protobuf::Timestamp
+      def self.timestamp(time) = Google::Protobuf::Timestamp.from_time(time)
+
+      # @param schedule [Proto::V1::Schedule, nil] the schedule
+      # @param random [Random] source of jitter
+      # @return [Integer] seconds of jitter
+      #: (Proto::V1::Schedule? schedule, Random random) -> Integer
+      private_class_method def self.jitter(schedule, random)
+        max = schedule&.jitter_seconds || 0
+        max.zero? ? 0 : random.rand(max + 1)
+      end
+
+      # @param value [Integer, nil] configured value
+      # @param default [Integer] value used when unset or 0
+      # @return [Integer] the value
+      #: (Integer? value, Integer default) -> Integer
+      private_class_method def self.nonzero(value, default) = value.nil? || value.zero? ? default : value
+    end
+  end
+end

data/lib/glazed/orchestrator.rb

@@ -0,0 +1,288 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'securerandom'
+require 'socket'
+require 'glazed'
+require 'glazed/must'
+require 'glazed/orchestrator/policy'
+require 'glazed/proto'
+require 'glazed/runner'
+require 'glazed/state_backends/base'
+require 'glazed/workflow'
+
+module Glazed
+  # Runs workflows for nodes as they become due, until stopped. Due dates
+  # live in a state backend, per workflow and node: nodes new to the inventory
+  # are due right away, signals move a node's due date per
+  # {Proto::V1::Debounce}, and each run schedules the next per
+  # {Proto::V1::Schedule}. Due nodes of a workflow are leased and run together
+  # as one batch, so batch steps such as publish see all of them. Workflows
+  # take turns, running one batch at a time.
+  #
+  # @example
+  #   orchestrator = Glazed::Orchestrator.new(settings:, workflows: {'backup' => workflow}, backend:, inventory: -> { nodes }, jobs: 4)
+  #   trap('TERM') { orchestrator.stop }
+  #   orchestrator.run
+  class Orchestrator
+    # Seconds between checks for due nodes when the settings set none.
+    DEFAULT_POLL = 30 #: Integer
+    # Seconds between inventory reloads when the settings set none.
+    DEFAULT_INVENTORY_REFRESH = 300 #: Integer
+    # Seconds a lease lasts when the settings set none.
+    DEFAULT_LEASE = 3600 #: Integer
+
+    # Byte written to the wakeup pipe by {#stop}.
+    STOP = 's' #: String
+    # Byte written to the wakeup pipe by {#reload_inventory}.
+    RELOAD_INVENTORY = 'i' #: String
+
+    #: type inventory = ^() -> Array[Proto::V1::Node]
+
+    # Applies signals to the nodes' schedules in one change.
+    #
+    # @param backend [StateBackends::Base] where schedules are kept
+    # @param workflow [String] workflow name
+    # @param signals [Array<Proto::V1::Signal>] the signals
+    # @param debounce [Proto::V1::Debounce, nil] the debounce settings
+    # @param now [Time] current time, used for signals without a reception time
+    # @return [Hash{String => Proto::V1::NodeSchedule}] the resulting records, by node name
+    #: (backend: StateBackends::Base, workflow: String, signals: Array[Proto::V1::Signal], debounce: Proto::V1::Debounce?, ?now: Time) -> Hash[String, Proto::V1::NodeSchedule]
+    def self.signal(backend:, workflow:, signals:, debounce:, now: Time.now)
+      by_node = signals.group_by(&:node)
+      backend.update_all(workflow, by_node.keys) do |node, record|
+        by_node.fetch(node).inject(record) { |current, signal| Policy.signal(current, signal, workflow:, now:, debounce:) }
+      end
+    end
+
+    # @param settings [Proto::V1::Orchestrator] orchestrator settings
+    # @param workflows [Hash{String => Workflow}] the workflows the settings orchestrate, by name
+    # @param backend [StateBackends::Base] where schedules are kept
+    # @param inventory [Proc] loads the current nodes
+    # @param jobs [Integer] nodes processed in parallel
+    # @param listener [Proc] receives events of the orchestrator and the runs
+    # @param clock [Proc] returns the current time
+    # @param random [Random] source of jitter
+    # @raise [ArgumentError] when the workflows differ from those the settings orchestrate
+    #: (settings: Proto::V1::Orchestrator, workflows: Hash[String, Workflow], backend: StateBackends::Base, inventory: Orchestrator::inventory, jobs: Integer, ?listener: Runner::listener, ?clock: ^() -> Time, ?random: Random) -> void
+    def initialize(settings:, workflows:, backend:, inventory:, jobs:, listener: ->(_event) {}, clock: -> { Time.now }, random: Random.new)
+      unless workflows.keys.sort == settings.workflows.keys.sort
+        raise ArgumentError, "workflows #{workflows.keys.sort} differ from the orchestrated #{settings.workflows.keys.sort}"
+      end
+
+      @settings = settings #: Proto::V1::Orchestrator
+      @workflows = workflows #: Hash[String, Workflow]
+      @backend = backend #: StateBackends::Base
+      @inventory = inventory #: Orchestrator::inventory
+      @jobs = jobs #: Integer
+      @listener = listener #: Runner::listener
+      @clock = clock #: ^() -> Time
+      @random = random #: Random
+      @owner = "#{Socket.gethostname}:#{Process.pid}:#{SecureRandom.hex(4)}" #: String
+      @nodes = {} #: Hash[String, Proto::V1::Node]
+      @inventory_loaded_at = nil #: Time?
+      # A pipe, as a thread blocked in Queue#pop misses a push from a signal handler on that thread.
+      wakeup_reader, wakeup_writer = IO.pipe
+      @wakeup_reader = wakeup_reader #: IO
+      @wakeup_writer = wakeup_writer #: IO
+      @stopping = false #: bool
+    end
+
+    # Runs batches of due nodes until {#stop} is called. Loading the inventory
+    # the first time must succeed; later reload failures are reported and the
+    # previous inventory is kept.
+    #
+    # @return [void]
+    # @raise [Glazed::Error] when the first inventory load or the state backend fails
+    #: () -> void
+    def run
+      notify(level: 'info', message: 'orchestrator started', owner: @owner)
+      until @stopping
+        wait = tick
+        wait_for_wakeup(wait)
+      end
+      notify(level: 'info', message: 'orchestrator stopped')
+    end
+
+    # Makes {#run} return after the current batch. Safe to call from a signal handler.
+    #
+    # @return [void]
+    #: () -> void
+    def stop = wake(STOP)
+
+    # Makes {#run} reload the inventory before the next batch. Safe to call from a signal handler.
+    #
+    # @return [void]
+    #: () -> void
+    def reload_inventory = wake(RELOAD_INVENTORY)
+
+    # Reloads the inventory when it is time, then runs one batch of due nodes
+    # for each workflow in turn.
+    #
+    # @return [Float] seconds until the next check
+    # @raise [Glazed::Error] when the first inventory load or the state backend fails
+    #: () -> Float
+    def tick
+      now = @clock.call
+      loaded_at = @inventory_loaded_at
+      refresh_inventory(now) if loaded_at.nil? || now - loaded_at >= inventory_refresh_seconds
+      @workflows.each_key { |workflow| run_batch(workflow, claim(workflow, @clock.call)) }
+      next_wait
+    end
+
+    # @param workflow [String] workflow name
+    # @return [Proto::V1::Orchestration] how the workflow is orchestrated
+    #: (String workflow) -> Proto::V1::Orchestration
+    private def orchestration(workflow)
+      Must.because(@settings.workflows[workflow]) { 'initialize checks the workflows against the settings' }
+    end
+
+    # @return [Integer] seconds between inventory reloads
+    #: () -> Integer
+    private def inventory_refresh_seconds = @settings.inventory_refresh_seconds.zero? ? DEFAULT_INVENTORY_REFRESH : @settings.inventory_refresh_seconds
+
+    # @return [Integer] seconds between checks for due nodes
+    #: () -> Integer
+    private def poll_seconds = @settings.poll_seconds.zero? ? DEFAULT_POLL : @settings.poll_seconds
+
+    # @param request [String] one byte naming the request
+    # @return [void]
+    #: (String request) -> void
+    private def wake(request)
+      @wakeup_writer.write_nonblock(request, exception: false)
+      nil
+    end
+
+    # Waits for a request from {#stop} or {#reload_inventory}, or until the timeout.
+    #
+    # @param timeout [Float] seconds to wait at most
+    # @return [void]
+    #: (Float timeout) -> void
+    private def wait_for_wakeup(timeout)
+      return unless @wakeup_reader.wait_readable(timeout)
+
+      requests = @wakeup_reader.readpartial(4096)
+      @stopping = true if requests.include?(STOP)
+      @inventory_loaded_at = nil if requests.include?(RELOAD_INVENTORY)
+    end
+
+    # Loads the inventory, and for each workflow adds records for new nodes and deletes those of nodes gone.
+    #
+    # @param now [Time] current time
+    # @return [void]
+    # @raise [StandardError] when loading the inventory fails the first time
+    #: (Time now) -> void
+    private def refresh_inventory(now)
+      begin
+        nodes = @inventory.call
+      rescue StandardError => e
+        raise if @inventory_loaded_at.nil?
+
+        # A transient failure of an inventory source must not stop scheduled runs.
+        notify(level: 'error', error: e.class.name, message: "inventory reload failed, keeping the previous one: #{e.message}")
+        @inventory_loaded_at = now
+        return
+      end
+
+      @nodes = nodes.to_h { [it.name, it] }
+      @inventory_loaded_at = now
+      notify(level: 'info', message: 'inventory loaded', nodes: @nodes.size)
+      @workflows.each_key do |workflow|
+        known = @backend.list(workflow).map(&:node)
+        added = @nodes.keys - known
+        removed = known - @nodes.keys
+        next if added.empty? && removed.empty?
+
+        schedule = orchestration(workflow).schedule
+        @backend.update_all(workflow, added + removed) do |node, record|
+          next nil unless @nodes.key?(node)
+
+          record || Policy.discovered(workflow, node, now:, schedule:, random: @random)
+        end
+        notify(level: 'info', workflow:, message: 'inventory changed', nodes: @nodes.size, added:, removed:)
+      end
+    end
+
+    # Leases due nodes of a workflow.
+    #
+    # @param workflow [String] workflow name
+    # @param now [Time] current time
+    # @return [Array(String, Array<String>)] identifier of the batch, and names of the leased nodes
+    #: (String workflow, Time now) -> [String, Array[String]]
+    private def claim(workflow, now)
+      owner = "#{@owner}:#{SecureRandom.hex(4)}"
+      settings = orchestration(workflow)
+      limit = settings.batch_size.zero? ? nil : settings.batch_size
+      due = @backend.list_due(workflow, now:, limit:)
+      return [owner, []] if due.empty?
+
+      lease_seconds = settings.lease_seconds.zero? ? DEFAULT_LEASE : settings.lease_seconds
+      records = @backend.update_all(workflow, due.map(&:node)) do |_node, record|
+        Policy.claim(record, owner:, now:, lease_seconds:)
+      end
+      [owner, records.values.select { it.lease_owner == owner }.map(&:node)]
+    end
+
+    # Runs the workflow for leased nodes and records the outcomes. When
+    # interrupted, the leases are released so the nodes run again soon.
+    #
+    # @param workflow [String] workflow name
+    # @param claimed [Array(String, Array<String>)] identifier of the batch, and names of the leased nodes
+    # @return [void]
+    # @raise [Interrupt] when the run is interrupted
+    #: (String workflow, [String, Array[String]] claimed) -> void
+    private def run_batch(workflow, claimed)
+      owner, names = claimed
+      return if names.empty?
+
+      nodes = names.filter_map { @nodes[it] }
+      notify(level: 'info', workflow:, message: 'batch started', nodes: nodes.map(&:name))
+      begin
+        errors = run_workflow(workflow, nodes)
+      rescue Interrupt
+        @backend.update_all(workflow, names) { |_node, record| Policy.released(record, owner:) }
+        raise
+      end
+      finished_at = @clock.call
+      schedule = orchestration(workflow).schedule
+      @backend.update_all(workflow, names) do |node, record|
+        next nil unless @nodes.key?(node)
+
+        Policy.completed(record, owner:, finished_at:, error: errors[node], schedule:, random: @random)
+      end
+      notify(level: 'info', workflow:, message: 'batch finished', nodes: nodes.size, failed: errors.size)
+    end
+
+    # Runs a workflow. A batch step failure fails every node of the batch.
+    #
+    # @param workflow [String] workflow name
+    # @param nodes [Array<Proto::V1::Node>] the nodes
+    # @return [Hash{String => String}] errors of the failed nodes, by node name
+    #: (String workflow, Array[Proto::V1::Node] nodes) -> Hash[String, String]
+    private def run_workflow(workflow, nodes)
+      result = Runner.new(workflow: @workflows.fetch(workflow), jobs: @jobs, listener: @listener).call(nodes)
+      result.failures.to_h { [it.node.name, "#{it.step}: #{it.error.message}"] }
+    rescue StandardError => e
+      # The nodes are scheduled for retry instead of the orchestrator stopping.
+      notify(level: 'error', workflow:, error: e.class.name, message: "batch failed: #{e.message}", backtrace: e.backtrace || [])
+      nodes.to_h { [it.name, "#{e.class}: #{e.message}"] }
+    end
+
+    # @return [Float] seconds until the earliest known due date of any workflow, at most the poll interval
+    #: () -> Float
+    private def next_wait
+      now = @clock.call
+      upcoming = @workflows.keys.filter_map { @backend.list_due(it, now: now + poll_seconds, limit: 1).first&.due_at&.to_time }.min
+      return poll_seconds.to_f unless upcoming
+
+      (upcoming - now).clamp(0.0, poll_seconds.to_f).to_f
+    end
+
+    # @param event [Hash{Symbol => String, Integer, Boolean, Array<String>, nil}] event fields
+    # @return [void]
+    #: (**Runner::event_value event) -> void
+    private def notify(**event) = @listener.call(event)
+  end
+end