glazed 0.1.0 → 0.2.0

data/lib/glazed/coordinator.rb

@@ -0,0 +1,252 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'securerandom'
+require 'time'
+require 'socket'
+require 'glazed'
+require 'glazed/must'
+require 'glazed/orchestrator/policy'
+require 'glazed/proto'
+require 'glazed/runner'
+require 'glazed/state_backends/base'
+
+module Glazed
+  # Changes node schedules in a state backend as the orchestrated workflows
+  # run: follows the inventory, applies signals, leases due nodes and records
+  # the outcome of their runs. {Orchestrator} drives it in one process; an
+  # external coordinator such as AWS Step Functions drives it one call at a time.
+  #
+  # @example
+  #   coordinator = Glazed::Coordinator.new(settings: config.orchestrator, backend:)
+  #   coordinator.sync_inventory(nodes.map(&:name))
+  #   owner, names = coordinator.claim('backup')
+  #   coordinator.complete('backup', owner:, names:, errors: {})
+  class Coordinator
+    # Seconds a lease lasts when the settings set none.
+    DEFAULT_LEASE = 3600 #: Integer
+    # Seconds between checks for due nodes when the settings set none.
+    DEFAULT_POLL = 30 #: Integer
+
+    # @param settings [Proto::V1::Orchestrator] orchestrator settings
+    # @param backend [StateBackends::Base] where schedules are kept
+    # @param listener [Proc] receives events of schedule changes
+    # @param clock [Proc] returns the current time
+    # @param random [Random] source of jitter
+    # @param owner [String] identifies this process in leases
+    #: (settings: Proto::V1::Orchestrator, backend: StateBackends::Base, ?listener: Runner::listener, ?clock: ^() -> Time, ?random: Random, ?owner: String) -> void
+    def initialize(settings:, backend:, listener: ->(_event) {}, clock: -> { Time.now }, random: Random.new,
+                   owner: "#{Socket.gethostname}:#{Process.pid}:#{SecureRandom.hex(4)}")
+      @settings = settings #: Proto::V1::Orchestrator
+      @backend = backend #: StateBackends::Base
+      @listener = listener #: Runner::listener
+      @clock = clock #: ^() -> Time
+      @random = random #: Random
+      @owner = owner #: String
+    end
+
+    # @return [Proto::V1::Orchestrator] orchestrator settings
+    #: Proto::V1::Orchestrator
+    attr_reader :settings
+    # @return [StateBackends::Base] where schedules are kept
+    #: StateBackends::Base
+    attr_reader :backend
+    # @return [String] identifies this process in leases
+    #: String
+    attr_reader :owner
+
+    # @return [Array<String>] names of the orchestrated workflows
+    #: () -> Array[String]
+    def workflows = @settings.workflows.keys
+
+    # @return [Integer] seconds between checks for due nodes
+    #: () -> Integer
+    def poll_seconds = @settings.poll_seconds.zero? ? DEFAULT_POLL : @settings.poll_seconds
+
+    # @return [Integer] the longest seconds a lease of any workflow lasts
+    #: () -> Integer
+    def longest_lease = @settings.workflows.values.map { lease_seconds(it) }.max || DEFAULT_LEASE
+
+    # @param workflow [String] workflow name
+    # @return [Proto::V1::Orchestration] how the workflow is orchestrated
+    # @raise [ArgumentError] when the workflow is not orchestrated
+    #: (String workflow) -> Proto::V1::Orchestration
+    def orchestration(workflow)
+      @settings.workflows[workflow] or raise ArgumentError, "workflow #{workflow.inspect} is not orchestrated"
+    end
+
+    # Adds records for nodes new to the inventory, due within the schedule's
+    # jitter, and deletes those of nodes gone, in every workflow.
+    #
+    # @param nodes [Array<String>] names of every node in the inventory
+    # @return [void]
+    #: (Array[String] nodes) -> void
+    def sync_inventory(nodes)
+      now = @clock.call
+      workflows.each do |workflow|
+        known = @backend.list(workflow).map(&:node)
+        added = nodes - known
+        removed = known - nodes
+        next if added.empty? && removed.empty?
+
+        schedule = orchestration(workflow).schedule
+        @backend.update_all(workflow, added + removed) do |node, record|
+          next nil if removed.include?(node)
+
+          record || Orchestrator::Policy.discovered(workflow, node, now:, schedule:, random: @random)
+        end
+        notify(level: 'info', workflow:, message: 'inventory changed', nodes: nodes.size, added:, removed:)
+      end
+    end
+
+    # Applies signals with each workflow's debounce, one change per workflow.
+    # Signals naming a node or workflow unknown here are reported and skipped.
+    #
+    # @param signals [Array<Proto::V1::Signal>] the signals
+    # @param nodes [Array<String>] names of every node in the inventory, which all_nodes signals stand for
+    # @return [Hash{String => Hash{String => Proto::V1::NodeSchedule}}] the resulting records, by workflow and node name
+    #: (Array[Proto::V1::Signal] signals, nodes: Array[String]) -> Hash[String, Hash[String, Proto::V1::NodeSchedule]]
+    def signal(signals, nodes:)
+      now = @clock.call
+      pending = {} #: Hash[String, Hash[String, Array[Proto::V1::Signal]]]
+      signals.each do |signal|
+        targets = signal.all_nodes ? nodes : [signal.node]
+        unknown_nodes = targets - nodes
+        targets -= unknown_nodes
+        notify(level: 'warn', message: 'signal for unknown nodes skipped', source: signal.source, nodes: unknown_nodes) unless unknown_nodes.empty?
+
+        chosen = signal.workflows.empty? ? workflows : signal.workflows.to_a
+        unknown_workflows = chosen - workflows
+        unless unknown_workflows.empty?
+          notify(level: 'warn', message: 'signal for workflows not orchestrated skipped', source: signal.source, workflows: unknown_workflows)
+        end
+
+        (chosen - unknown_workflows).product(targets).each do |workflow, node|
+          ((pending[workflow] ||= {})[node] ||= []) << for_node(signal, node)
+        end
+      end
+
+      pending.to_h do |workflow, by_node|
+        debounce = orchestration(workflow).debounce
+        records = @backend.update_all(workflow, by_node.keys) do |node, record|
+          by_node.fetch(node).inject(record) { |current, signal| Orchestrator::Policy.signal(current, signal, workflow:, now:, debounce:) }
+        end
+        [workflow, records]
+      end
+    end
+
+    # Leases due nodes of a workflow, at most the workflow's batch size.
+    #
+    # @param workflow [String] workflow name
+    # @return [Array(String, Array<String>)] identifier of the batch, and names of the leased nodes
+    #: (String workflow) -> [String, Array[String]]
+    def claim(workflow)
+      now = @clock.call
+      owner = "#{@owner}:#{SecureRandom.hex(4)}"
+      settings = orchestration(workflow)
+      limit = settings.batch_size.zero? ? nil : settings.batch_size
+      due = @backend.list_due(workflow, now:, limit:)
+      return [owner, []] if due.empty?
+
+      lease_seconds = lease_seconds(settings)
+      records = @backend.update_all(workflow, due.map(&:node)) do |_node, record|
+        Orchestrator::Policy.claim(record, owner:, now:, lease_seconds:)
+      end
+      [owner, records.values.select { it.lease_owner == owner }.map(&:node)]
+    end
+
+    # Records the outcome of a batch and schedules each node's next run.
+    # Records leased to another batch, and records deleted meanwhile, stay as they are.
+    #
+    # @param workflow [String] workflow name
+    # @param owner [String] identifier of the batch
+    # @param names [Array<String>] names of the nodes the batch ran
+    # @param errors [Hash{String => String}] why nodes failed, by node name
+    # @return [void]
+    #: (String workflow, owner: String, names: Array[String], errors: Hash[String, String]) -> void
+    def complete(workflow, owner:, names:, errors:)
+      finished_at = @clock.call
+      schedule = orchestration(workflow).schedule
+      @backend.update_all(workflow, names) do |node, record|
+        Orchestrator::Policy.completed(record, owner:, finished_at:, error: errors[node], schedule:, random: @random)
+      end
+    end
+
+    # Gives up leases without recording a run, so the nodes run again soon.
+    #
+    # @param workflow [String] workflow name
+    # @param owner [String] identifier of the batch
+    # @param names [Array<String>] names of the leased nodes
+    # @return [void]
+    #: (String workflow, owner: String, names: Array[String]) -> void
+    def release(workflow, owner:, names:)
+      @backend.update_all(workflow, names) { |_node, record| Orchestrator::Policy.released(record, owner:) }
+    end
+
+    # The earliest time a node of any workflow may be claimed: its due date,
+    # or the end of its lease while another batch holds it.
+    #
+    # @return [Time, nil] the time; nil when there are no records
+    #: () -> Time?
+    def next_due
+      now = @clock.call
+      times = workflows.flat_map do |workflow|
+        @backend.list(workflow).filter_map do |record|
+          Orchestrator::Policy.time(Orchestrator::Policy.running?(record, now) ? record.lease_until : record.due_at)
+        end
+      end
+      times.min
+    end
+
+    # Decides whether a coordinator execution waits for the next due node,
+    # recording it as the waiter, or ends because another execution already
+    # waits until then. See {Orchestrator::Policy.arm}.
+    #
+    # @param token [String] identifies the execution
+    # @param cycle [Integer] times the execution has run due nodes already
+    # @param max_cycles [Integer] cycles after which the execution ends
+    # @param grace [Integer, nil] seconds after waking until another execution may take over; {#longest_lease} when nil
+    # @return [Time, nil] when to wake up; nil when the execution ends
+    #: (token: String, cycle: Integer, max_cycles: Integer, ?grace: Integer?) -> Time?
+    def arm(token:, cycle:, max_cycles:, grace: nil)
+      next_due = self.next_due
+      now = @clock.call
+      wake = nil #: Time?
+      @backend.update_wakeup do |record|
+        kept, wake = Orchestrator::Policy.arm(record, token:, next_due:, now:, gap: poll_seconds, grace: grace || longest_lease, cycle:, max_cycles:)
+        kept
+      end
+      if wake
+        notify(level: 'info', message: 'waiting for due nodes', token:, cycle:, wait_until: wake.utc.iso8601)
+      else
+        notify(level: 'info', message: 'not waiting', token:, cycle:)
+      end
+      wake
+    end
+
+    # @param settings [Proto::V1::Orchestration] how a workflow is orchestrated
+    # @return [Integer] seconds a lease of the workflow lasts
+    #: (Proto::V1::Orchestration settings) -> Integer
+    private def lease_seconds(settings) = settings.lease_seconds.zero? ? DEFAULT_LEASE : settings.lease_seconds
+
+    # @param signal [Proto::V1::Signal] a signal for one node or every node
+    # @param node [String] node name
+    # @return [Proto::V1::Signal] the signal for that node alone
+    #: (Proto::V1::Signal signal, String node) -> Proto::V1::Signal
+    private def for_node(signal, node)
+      return signal unless signal.all_nodes
+
+      signal.dup.tap do |copy|
+        copy.node = node
+        copy.all_nodes = false
+      end
+    end
+
+    # @param event [Hash{Symbol => String, Integer, Boolean, Array<String>, nil}] event fields
+    # @return [void]
+    #: (**Runner::event_value event) -> void
+    private def notify(**event) = @listener.call(event)
+  end
+end

data/lib/glazed/dialect/run.rb

@@ -9,6 +9,7 @@ require 'glazed/dialect'
 require 'glazed/dialect/actions'
 require 'glazed/proto'
 require 'glazed/secret'
+require 'glazed/secrets_resolver'
 
 module Glazed
   module Dialect
@@ -17,11 +18,13 @@ module Glazed
       # @param cli [CliSession] session to the node
       # @param node [Proto::V1::Node] node being run against
       # @param dialect [Class<Dialects::Base>] dialect of the node
-      #: (cli: CliSession, node: Proto::V1::Node, dialect: singleton(Dialects::Base)) -> void
-      def initialize(cli:, node:, dialect:)
+      # @param secrets [SecretsResolver] reads the node's secrets
+      #: (cli: CliSession, node: Proto::V1::Node, dialect: singleton(Dialects::Base), ?secrets: SecretsResolver) -> void
+      def initialize(cli:, node:, dialect:, secrets: SecretsResolver.new)
         @cli = cli #: CliSession
         @node = node #: Proto::V1::Node
         @dialect = dialect #: singleton(Dialects::Base)
+        @secrets = secrets #: SecretsResolver
         @vars = {} #: Hash[Symbol, untyped]
         @output = +'' #: String
       end
@@ -32,6 +35,9 @@ module Glazed
       # @return [Proto::V1::Node] node being run against
       #: Proto::V1::Node
       attr_reader :node
+      # @return [SecretsResolver] reads the node's secrets
+      #: SecretsResolver
+      attr_reader :secrets
       # @return [Class<Dialects::Base>] dialect of the node
       #: singleton(Dialects::Base)
       attr_reader :dialect
@@ -78,7 +84,7 @@ module Glazed
       # @param name [String] secret name
       # @return [Secret, nil] the node's secret
       #: (String name) -> Secret?
-      def secret(name) = Secret.maybe(node.secrets[name])
+      def secret(name) = secrets.maybe(node.secrets[name])
     end
   end
 end

data/lib/glazed/dialect/sensitive_pattern.rb

@@ -14,10 +14,12 @@ module Glazed
       # @param pattern [Regexp] pattern matched against the whole configuration
       # @param determiner [Proc] returns the secret within a match, or its offsets in the configuration,
       #   or nil when the match holds none
-      #: (pattern: Regexp, determiner: SensitivePattern::determiner) -> void
-      def initialize(pattern:, determiner:)
+      # @param note [String, nil] shown in the replacement of every secret this pattern locates
+      #: (pattern: Regexp, determiner: SensitivePattern::determiner, ?note: String?) -> void
+      def initialize(pattern:, determiner:, note: nil)
         @pattern = pattern #: Regexp
         @determiner = determiner #: SensitivePattern::determiner
+        @note = note #: String?
       end
 
       # @return [Regexp] pattern matched against the whole configuration
@@ -26,6 +28,9 @@ module Glazed
       # @return [Proc] returns the secret within a match, or its offsets
       #: SensitivePattern::determiner
       attr_reader :determiner
+      # @return [String, nil] shown in the replacement of every secret this pattern locates
+      #: String?
+      attr_reader :note
     end
   end
 end

data/lib/glazed/orchestrator/policy.rb

@@ -146,6 +146,34 @@ module Glazed
         end
       end
 
+      # Decides whether a coordinator execution waits for the next due node.
+      # It does unless another live execution already waits until then, or
+      # it has looped max_cycles times; a waiter that stops waiting gives up
+      # its record.
+      #
+      # @param record [Proto::V1::Wakeup, nil] the current waiter
+      # @param token [String] identifies the deciding execution
+      # @param next_due [Time, nil] when a node may next be claimed; nil when never
+      # @param now [Time] current time
+      # @param gap [Integer] seconds the execution waits at least after its first cycle
+      # @param grace [Integer] seconds after waking until another execution may take over
+      # @param cycle [Integer] times the execution has run due nodes already
+      # @param max_cycles [Integer] cycles after which the execution ends
+      # @return [Array(Proto::V1::Wakeup, Time), Array(Proto::V1::Wakeup, nil), Array(nil, nil)] the waiter
+      #   record to keep, and when to wake; nil when the execution ends
+      #: (Proto::V1::Wakeup? record, token: String, next_due: Time?, now: Time, gap: Integer, grace: Integer, cycle: Integer, max_cycles: Integer) -> [Proto::V1::Wakeup?, Time?]
+      def self.arm(record, token:, next_due:, now:, gap:, grace:, cycle:, max_cycles:)
+        own = record&.owner == token
+        return [own ? nil : record, nil] if next_due.nil? || cycle >= max_cycles
+
+        wake = [next_due, cycle.zero? ? now : now + gap].max
+        expires_at = time(record&.expires_at)
+        wake_at = time(record&.wake_at)
+        return [record, nil] if !own && expires_at && expires_at > now && wake_at && wake_at <= wake
+
+        [Proto::V1::Wakeup.new(owner: token, wake_at: timestamp(wake), expires_at: timestamp(wake + grace)), wake]
+      end
+
       # @param record [Proto::V1::NodeSchedule] the node's record
       # @param now [Time] current time
       # @return [Boolean] whether an orchestrator holds an unexpired lease on the node

data/lib/glazed/orchestrator.rb

@@ -3,10 +3,8 @@
 
 # SPDX-License-Identifier: Apache-2.0
 
-require 'securerandom'
-require 'socket'
 require 'glazed'
-require 'glazed/must'
+require 'glazed/coordinator'
 require 'glazed/orchestrator/policy'
 require 'glazed/proto'
 require 'glazed/runner'
@@ -28,12 +26,9 @@ module Glazed
   #   orchestrator.run
   class Orchestrator
     # Seconds between checks for due nodes when the settings set none.
-    DEFAULT_POLL = 30 #: Integer
+    DEFAULT_POLL = Coordinator::DEFAULT_POLL #: Integer
     # Seconds between inventory reloads when the settings set none.
     DEFAULT_INVENTORY_REFRESH = 300 #: Integer
-    # Seconds a lease lasts when the settings set none.
-    DEFAULT_LEASE = 3600 #: Integer
-
     # Byte written to the wakeup pipe by {#stop}.
     STOP = 's' #: String
     # Byte written to the wakeup pipe by {#reload_inventory}.
@@ -49,8 +44,11 @@ module Glazed
     # @param debounce [Proto::V1::Debounce, nil] the debounce settings
     # @param now [Time] current time, used for signals without a reception time
     # @return [Hash{String => Proto::V1::NodeSchedule}] the resulting records, by node name
+    # @raise [ArgumentError] for a signal of every node, which {Coordinator#signal} expands
     #: (backend: StateBackends::Base, workflow: String, signals: Array[Proto::V1::Signal], debounce: Proto::V1::Debounce?, ?now: Time) -> Hash[String, Proto::V1::NodeSchedule]
     def self.signal(backend:, workflow:, signals:, debounce:, now: Time.now)
+      raise ArgumentError, 'expand all_nodes signals with Coordinator#signal' if signals.any?(&:all_nodes)
+
       by_node = signals.group_by(&:node)
       backend.update_all(workflow, by_node.keys) do |node, record|
         by_node.fetch(node).inject(record) { |current, signal| Policy.signal(current, signal, workflow:, now:, debounce:) }
@@ -74,13 +72,11 @@ module Glazed
 
       @settings = settings #: Proto::V1::Orchestrator
       @workflows = workflows #: Hash[String, Workflow]
-      @backend = backend #: StateBackends::Base
+      @coordinator = Coordinator.new(settings:, backend:, listener:, clock:, random:) #: Coordinator
       @inventory = inventory #: Orchestrator::inventory
       @jobs = jobs #: Integer
       @listener = listener #: Runner::listener
       @clock = clock #: ^() -> Time
-      @random = random #: Random
-      @owner = "#{Socket.gethostname}:#{Process.pid}:#{SecureRandom.hex(4)}" #: String
       @nodes = {} #: Hash[String, Proto::V1::Node]
       @inventory_loaded_at = nil #: Time?
       # A pipe, as a thread blocked in Queue#pop misses a push from a signal handler on that thread.
@@ -98,7 +94,7 @@ module Glazed
     # @raise [Glazed::Error] when the first inventory load or the state backend fails
     #: () -> void
     def run
-      notify(level: 'info', message: 'orchestrator started', owner: @owner)
+      notify(level: 'info', message: 'orchestrator started', owner: @coordinator.owner)
       until @stopping
         wait = tick
         wait_for_wakeup(wait)
@@ -128,24 +124,17 @@ module Glazed
       now = @clock.call
       loaded_at = @inventory_loaded_at
       refresh_inventory(now) if loaded_at.nil? || now - loaded_at >= inventory_refresh_seconds
-      @workflows.each_key { |workflow| run_batch(workflow, claim(workflow, @clock.call)) }
+      @workflows.each_key { |workflow| run_batch(workflow, @coordinator.claim(workflow)) }
       next_wait
     end
 
-    # @param workflow [String] workflow name
-    # @return [Proto::V1::Orchestration] how the workflow is orchestrated
-    #: (String workflow) -> Proto::V1::Orchestration
-    private def orchestration(workflow)
-      Must.because(@settings.workflows[workflow]) { 'initialize checks the workflows against the settings' }
-    end
-
     # @return [Integer] seconds between inventory reloads
     #: () -> Integer
     private def inventory_refresh_seconds = @settings.inventory_refresh_seconds.zero? ? DEFAULT_INVENTORY_REFRESH : @settings.inventory_refresh_seconds
 
     # @return [Integer] seconds between checks for due nodes
     #: () -> Integer
-    private def poll_seconds = @settings.poll_seconds.zero? ? DEFAULT_POLL : @settings.poll_seconds
+    private def poll_seconds = @coordinator.poll_seconds
 
     # @param request [String] one byte naming the request
     # @return [void]
@@ -189,40 +178,7 @@ module Glazed
       @nodes = nodes.to_h { [it.name, it] }
       @inventory_loaded_at = now
       notify(level: 'info', message: 'inventory loaded', nodes: @nodes.size)
-      @workflows.each_key do |workflow|
-        known = @backend.list(workflow).map(&:node)
-        added = @nodes.keys - known
-        removed = known - @nodes.keys
-        next if added.empty? && removed.empty?
-
-        schedule = orchestration(workflow).schedule
-        @backend.update_all(workflow, added + removed) do |node, record|
-          next nil unless @nodes.key?(node)
-
-          record || Policy.discovered(workflow, node, now:, schedule:, random: @random)
-        end
-        notify(level: 'info', workflow:, message: 'inventory changed', nodes: @nodes.size, added:, removed:)
-      end
-    end
-
-    # Leases due nodes of a workflow.
-    #
-    # @param workflow [String] workflow name
-    # @param now [Time] current time
-    # @return [Array(String, Array<String>)] identifier of the batch, and names of the leased nodes
-    #: (String workflow, Time now) -> [String, Array[String]]
-    private def claim(workflow, now)
-      owner = "#{@owner}:#{SecureRandom.hex(4)}"
-      settings = orchestration(workflow)
-      limit = settings.batch_size.zero? ? nil : settings.batch_size
-      due = @backend.list_due(workflow, now:, limit:)
-      return [owner, []] if due.empty?
-
-      lease_seconds = settings.lease_seconds.zero? ? DEFAULT_LEASE : settings.lease_seconds
-      records = @backend.update_all(workflow, due.map(&:node)) do |_node, record|
-        Policy.claim(record, owner:, now:, lease_seconds:)
-      end
-      [owner, records.values.select { it.lease_owner == owner }.map(&:node)]
+      @coordinator.sync_inventory(@nodes.keys)
     end
 
     # Runs the workflow for leased nodes and records the outcomes. When
@@ -242,16 +198,11 @@ module Glazed
       begin
         errors = run_workflow(workflow, nodes)
       rescue Interrupt
-        @backend.update_all(workflow, names) { |_node, record| Policy.released(record, owner:) }
+        @coordinator.release(workflow, owner:, names:)
         raise
       end
-      finished_at = @clock.call
-      schedule = orchestration(workflow).schedule
-      @backend.update_all(workflow, names) do |node, record|
-        next nil unless @nodes.key?(node)
-
-        Policy.completed(record, owner:, finished_at:, error: errors[node], schedule:, random: @random)
-      end
+      # Records of nodes gone from the inventory are deleted by the next reload.
+      @coordinator.complete(workflow, owner:, names: names & @nodes.keys, errors:)
       notify(level: 'info', workflow:, message: 'batch finished', nodes: nodes.size, failed: errors.size)
     end
 
@@ -273,11 +224,10 @@ module Glazed
     # @return [Float] seconds until the earliest known due date of any workflow, at most the poll interval
     #: () -> Float
     private def next_wait
-      now = @clock.call
-      upcoming = @workflows.keys.filter_map { @backend.list_due(it, now: now + poll_seconds, limit: 1).first&.due_at&.to_time }.min
+      upcoming = @coordinator.next_due
       return poll_seconds.to_f unless upcoming
 
-      (upcoming - now).clamp(0.0, poll_seconds.to_f).to_f
+      (upcoming - @clock.call).clamp(0.0, poll_seconds.to_f).to_f
     end
 
     # @param event [Hash{Symbol => String, Integer, Boolean, Array<String>, nil}] event fields

data/lib/glazed/proto.rb

@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 require 'glazed/v1/config_pb'
+require 'glazed/v1/coordinator_pb'
 require 'glazed/v1/inventory_pb'
 require 'glazed/v1/orchestrator_pb'
 require 'glazed/v1/redaction_pb'