glazed 0.1.0

data/lib/glazed/sources/base.rb

@@ -0,0 +1,79 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'json'
+require 'google/protobuf'
+require 'glazed'
+require 'glazed/proto'
+require 'glazed/proto_json'
+require 'glazed/validation'
+
+module Glazed
+  module Sources
+    # Inventory provider. Subclasses implement {.from_proto} and {#nodes}.
+    class Base
+      # Raised when inventory is malformed or a node lacks a name or dialect.
+      class InvalidInventory < Glazed::Error; end
+
+      # Node fields merged key by key with defaults.
+      MAP_FIELDS = %w[params secrets].freeze #: Array[String]
+
+      # Builds a provider from its settings message.
+      #
+      # @param settings [Google::Protobuf::MessageExts] provider settings from {Proto::V1::InventorySource}
+      # @return [Base] the provider
+      #: (Google::Protobuf::MessageExts settings) -> Base
+      def self.from_proto(settings) = raise NotImplementedError
+
+      # Reads the nodes.
+      #
+      # @return [Array<Proto::V1::Node>] nodes, each with a name and a dialect
+      #: () -> Array[Proto::V1::Node]
+      def nodes = raise NotImplementedError
+
+      # @return [String] where the nodes come from, for error messages
+      #: () -> String
+      def description = self.class.name.to_s
+
+      # Builds the nodes of ProtoJSON of an {Proto::V1::InventoryFile}.
+      #
+      # @param json [String] the inventory file
+      # @param origin [String] where the JSON comes from, for error messages
+      # @return [Array<Proto::V1::Node>] nodes with defaults applied
+      # @raise [InvalidInventory] when the JSON is malformed
+      # @raise [InvalidMessage] when the inventory breaks the rules of its schema
+      #: (String json, origin: String) -> Array[Proto::V1::Node]
+      private def nodes_from_json(json, origin:)
+        Validation.validate!(ProtoJson.load(Proto::V1::InventoryFile, json), origin:)
+        document = JSON.parse(json) #: Hash[String, untyped]
+        nodes_from(document, origin:)
+      rescue Google::Protobuf::ParseError, JSON::ParserError => e
+        raise InvalidInventory, "#{origin}: #{e.message}"
+      end
+
+      # Builds the nodes of an {Proto::V1::InventoryFile} document, filling
+      # fields each node leaves unset from its defaults. The params and secrets
+      # maps are merged key by key.
+      #
+      # @param document [Hash{String => Object}] the inventory file as decoded JSON
+      # @param origin [String] where the document comes from, for error messages
+      # @return [Array<Proto::V1::Node>] the nodes
+      # @raise [InvalidInventory] when a node is malformed
+      #: (Hash[String, untyped] document, origin: String) -> Array[Proto::V1::Node]
+      private def nodes_from(document, origin:)
+        defaults = document.fetch('defaults', {}) #: Hash[String, untyped]
+        nodes = document.fetch('nodes', []) #: Array[Hash[String, untyped]]
+        nodes.map do |attributes|
+          merged = defaults.merge(attributes) do |key, default, value|
+            MAP_FIELDS.include?(key) ? default.merge(value) : value
+          end
+          ProtoJson.load(Proto::V1::Node, JSON.generate(merged))
+        rescue Google::Protobuf::ParseError => e
+          raise InvalidInventory, "#{origin}: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/glazed/sources/command.rb

@@ -0,0 +1,67 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'glazed'
+require 'glazed/must'
+require 'glazed/proto'
+require 'glazed/shell_command'
+require 'glazed/sources'
+require 'glazed/sources/base'
+
+module Glazed
+  module Sources
+    # Inventory printed by a command as ProtoJSON of {Proto::V1::InventoryFile},
+    # for nodes kept in another system.
+    #
+    # @example
+    #   Glazed::Sources::Command.new(run: 'bin/inventory --site hnd').nodes
+    class Command < Base
+      # Seconds before the command is killed when the settings set none.
+      DEFAULT_TIMEOUT = 300 #: Integer
+
+      # @param run [String] command line, run by /bin/sh -c
+      # @param timeout [Numeric] seconds before the command is killed
+      #: (run: String, ?timeout: Numeric) -> void
+      def initialize(run:, timeout: DEFAULT_TIMEOUT)
+        super()
+        @run = run #: String
+        @timeout = timeout #: Numeric
+      end
+
+      # Builds a provider from {Proto::V1::CommandInventory}.
+      #
+      # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::CommandInventory}
+      # @return [Command] the provider
+      # @raise [Must::Violation] when settings is another message
+      #: (Google::Protobuf::MessageExts settings) -> Command
+      def self.from_proto(settings)
+        command = Must.be_a(settings, Proto::V1::CommandInventory)
+        new(run: command.run, timeout: command.timeout_seconds.zero? ? DEFAULT_TIMEOUT : command.timeout_seconds)
+      end
+
+      # @return [String] command line
+      #: String
+      attr_reader :run
+
+      # @return [Numeric] seconds before the command is killed
+      #: Numeric
+      attr_reader :timeout
+
+      # Runs the command and reads the nodes it prints.
+      #
+      # @return [Array<Proto::V1::Node>] nodes with defaults applied
+      # @raise [ShellCommand::Failed] when the command exits non-zero
+      # @raise [TimeoutError] when the command does not finish in time
+      # @raise [InvalidInventory] when the output is not an inventory file
+      # @raise [InvalidMessage] when the output breaks the rules of its schema
+      #: () -> Array[Proto::V1::Node]
+      def nodes = nodes_from_json(ShellCommand.new(run:, env: {}, stdin: '', timeout:).call, origin: description)
+
+      # @return [String] the command line
+      #: () -> String
+      def description = "command #{run}"
+    end
+  end
+end

data/lib/glazed/sources/inline.rb

@@ -0,0 +1,56 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'json'
+require 'glazed/must'
+require 'glazed/proto'
+require 'glazed/proto_json'
+require 'glazed/sources'
+require 'glazed/sources/base'
+require 'glazed/validation'
+
+module Glazed
+  module Sources
+    # Inventory written in the config itself, for setups that keep everything
+    # in one file. Defaults apply as in {JsonFiles}.
+    #
+    # @example Inventory source
+    #   {"inline": {"defaults": {"dialect": "ios"}, "nodes": [{"name": "rt1"}]}}
+    class Inline < Base
+      # @param inventory [Proto::V1::InventoryFile] nodes and their defaults
+      #: (inventory: Proto::V1::InventoryFile) -> void
+      def initialize(inventory:)
+        super()
+        @inventory = inventory #: Proto::V1::InventoryFile
+      end
+
+      # Builds a provider from an inline {Proto::V1::InventoryFile}.
+      #
+      # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::InventoryFile}
+      # @return [Inline] the provider
+      # @raise [Must::Violation] when settings is another message
+      # @raise [InvalidMessage] when the inventory breaks the rules of its schema
+      #: (Google::Protobuf::MessageExts settings) -> Inline
+      def self.from_proto(settings)
+        inventory = Must.be_a(settings, Proto::V1::InventoryFile)
+        Validation.validate!(inventory, origin: 'inline inventory')
+        new(inventory:)
+      end
+
+      # @return [Array<Proto::V1::Node>] the nodes with defaults applied
+      # @raise [InvalidInventory] when a node lacks a name or dialect
+      #: () -> Array[Proto::V1::Node]
+      def nodes
+        # ProtoJSON omits unset fields, so defaults merge as they do for files.
+        document = JSON.parse(ProtoJson.dump(@inventory)) #: Hash[String, untyped]
+        nodes_from(document, origin: description)
+      end
+
+      # @return [String] where the nodes come from
+      #: () -> String
+      def description = 'inline inventory'
+    end
+  end
+end

data/lib/glazed/sources/json_files.rb

@@ -0,0 +1,87 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'glazed'
+require 'glazed/must'
+require 'glazed/proto'
+require 'glazed/sources'
+require 'glazed/sources/base'
+
+module Glazed
+  module Sources
+    # Inventory read from ProtoJSON files of {Proto::V1::InventoryFile}.
+    #
+    # Each file's `defaults` fill fields its nodes leave unset. The `params`
+    # and `secrets` maps are merged key by key.
+    #
+    # @example
+    #   Glazed::Sources::JsonFiles.new(paths: ['inventory/*.json']).nodes
+    class JsonFiles < Base
+      # Raised when a file is not a valid inventory file.
+      InvalidFile = Base::InvalidInventory
+
+      # @param paths [Array<String>] file paths or glob patterns
+      #: (paths: Array[String]) -> void
+      def initialize(paths:)
+        super()
+        @paths = paths #: Array[String]
+      end
+
+      # Builds a provider from {Proto::V1::JsonFilesInventory}.
+      #
+      # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::JsonFilesInventory}
+      # @return [JsonFiles] the provider
+      # @raise [Must::Violation] when settings is another message
+      #: (Google::Protobuf::MessageExts settings) -> JsonFiles
+      def self.from_proto(settings)
+        new(paths: Must.be_a(settings, Proto::V1::JsonFilesInventory).paths.to_a)
+      end
+
+      # @return [Array<String>] file paths or glob patterns
+      #: Array[String]
+      attr_reader :paths
+
+      # Reads every file and returns its nodes.
+      #
+      # @return [Array<Proto::V1::Node>] nodes in file order
+      # @raise [InvalidFile] when a file is malformed or a node lacks a name or dialect
+      # @raise [Sources::DuplicateNode] when a node name appears twice
+      # @raise [Errno::ENOENT] when a literal path does not exist
+      #: () -> Array[Proto::V1::Node]
+      def nodes
+        origins = {} #: Hash[String, String]
+        files.flat_map do |path|
+          load_file(path).each do |node|
+            if (other = origins[node.name])
+              raise DuplicateNode, "node #{node.name.inspect} is defined in both #{other} and #{path}"
+            end
+
+            origins[node.name] = path
+          end
+        end
+      end
+
+      # @return [String] the path patterns
+      #: () -> String
+      def description = "json_files #{paths.join(', ')}"
+
+      # @return [Array<String>] files matched by {#paths}, in a stable order
+      #: () -> Array[String]
+      private def files
+        paths.flat_map do |pattern|
+          matches = Dir.glob(pattern).sort
+          matches.empty? && !pattern.match?(/[*?\[{]/) ? [pattern] : matches
+        end.uniq
+      end
+
+      # @param path [String] inventory file
+      # @return [Array<Proto::V1::Node>] nodes with defaults applied
+      # @raise [InvalidFile] when the file is malformed
+      # @raise [InvalidMessage] when the file breaks the rules of its schema
+      #: (String path) -> Array[Proto::V1::Node]
+      private def load_file(path) = nodes_from_json(File.read(path), origin: path)
+    end
+  end
+end

data/lib/glazed/sources.rb

@@ -0,0 +1,105 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'glazed'
+require 'glazed/finder'
+require 'glazed/must'
+require 'glazed/plugin_registry'
+require 'glazed/proto'
+require 'glazed/sources/base'
+require 'glazed/validation'
+
+module Glazed
+  # Namespace of inventory providers, looked up by name.
+  module Sources
+    # Raised when an inventory source names no provider.
+    class MissingProvider < Glazed::Error; end
+    # Raised when two nodes share a name.
+    class DuplicateNode < Glazed::Error; end
+
+    # Finds a provider class by snake_case name.
+    #
+    # @param name [String, Symbol] provider name, such as "json_files"
+    # @return [Class<Base>] the provider class
+    # @raise [Finder::NotFound] when there is none, for example when the gem providing it is not installed
+    # @raise [Must::Violation] when the name resolves to something other than a provider
+    #: (String | Symbol name) -> singleton(Base)
+    def self.find(name)
+      found = Finder.find(self, 'glazed/sources', name)
+      return found if found.is_a?(Class) && found < Base
+
+      raise Must::Violation, "#{found} is not an inventory provider"
+    end
+
+    # Inventory providers implemented by plugins, by settings message type.
+    PLUGINS = PluginRegistry.new('inventory provider') #: PluginRegistry
+
+    # Registers a plugin provider, selected by `plugin` sources whose settings are of the given type.
+    #
+    # @param settings_class [Class] generated message class of the provider settings
+    # @param implementation [Class<Base>] provider built with {Base.from_proto}
+    # @return [void]
+    # @example
+    #   Glazed::Sources.register(GlazedNetbox::V1::NetboxInventory, GlazedNetbox::Source)
+    #: (singleton(Google::Protobuf::AbstractMessage) settings_class, singleton(Base) implementation) -> void
+    def self.register(settings_class, implementation)
+      PLUGINS.register(settings_class, implementation)
+    end
+
+    # Builds the provider a source selects.
+    #
+    # @param source [Proto::V1::InventorySource] provider settings
+    # @return [Base] the provider
+    # @raise [InvalidMessage] when the source breaks the rules of its schema
+    # @raise [Finder::NotFound] when the provider is not available
+    # @raise [PluginRegistry::UnknownPlugin] when no plugin is registered for the settings type
+    #: (Proto::V1::InventorySource source) -> Base
+    def self.build(source)
+      Validation.validate!(source)
+      provider = source.provider or raise MissingProvider, 'inventory source sets no provider'
+      if (packed = source.plugin)
+        implementation, settings = PLUGINS.resolve(packed)
+        return provider_class(implementation).from_proto(settings)
+      end
+
+      settings = Must.be_a(source[provider.to_s], Google::Protobuf::MessageExts)
+      find(provider).from_proto(settings)
+    end
+
+    # @param implementation [Module] registered implementation
+    # @return [Class<Base>] the implementation as a provider class
+    # @raise [Must::Violation] when it is not a provider
+    #: (Module[untyped] implementation) -> singleton(Base)
+    private_class_method def self.provider_class(implementation)
+      return implementation if implementation.is_a?(Class) && implementation < Base
+
+      raise Must::Violation, "#{implementation} is not an inventory provider"
+    end
+
+    # Reads the nodes of every source.
+    #
+    # @param sources [Array<Proto::V1::InventorySource>] sources in order
+    # @return [Array<Proto::V1::Node>] nodes in source order
+    # @raise [DuplicateNode] when a node name appears twice
+    # @raise [Base::InvalidInventory] when a node lacks a name or dialect
+    # @raise [InvalidMessage] when a source breaks the rules of its schema
+    #: (Array[Proto::V1::InventorySource] sources) -> Array[Proto::V1::Node]
+    def self.nodes(sources)
+      origins = {} #: Hash[String, String]
+      sources.flat_map do |source|
+        provider = build(source)
+        provider.nodes.each do |node|
+          raise Base::InvalidInventory, "#{provider.description}: a node has no name" if node.name.empty?
+          raise Base::InvalidInventory, "#{provider.description}: node #{node.name.inspect} has no dialect" if node.dialect.empty?
+          if (other = origins[node.name])
+            raise DuplicateNode, "node #{node.name.inspect} is defined in both #{other} and #{provider.description}"
+          end
+
+          origins[node.name] = provider.description
+        end
+      end
+    end
+  end
+end

data/lib/glazed/state_backends/base.rb

@@ -0,0 +1,72 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'google/protobuf'
+require 'glazed'
+require 'glazed/proto'
+
+module Glazed
+  module StateBackends
+    # Store of {Proto::V1::NodeSchedule} records, one per workflow and node.
+    # Records change only through read-modify-write blocks, which a backend may
+    # run more than once when another process changed the records meanwhile,
+    # so a block must compute its result from the record it is given alone.
+    # Subclasses implement {.from_proto}, {#update_all}, {#list_due} and {#list}.
+    class Base
+      #: type updater = ^(String node, Proto::V1::NodeSchedule? record) -> Proto::V1::NodeSchedule?
+
+      # Builds a state backend from its settings message.
+      #
+      # @param settings [Google::Protobuf::MessageExts] backend settings from {Proto::V1::StateBackendConfig}
+      # @param context [StepContext] configured storages, and where progress is reported
+      # @return [Base] the state backend
+      #: (Google::Protobuf::MessageExts settings, context: StepContext) -> Base
+      def self.from_proto(settings, context:) = raise NotImplementedError
+
+      # Replaces the records of nodes with what the block returns.
+      #
+      # @param workflow [String] workflow name
+      # @param nodes [Array<String>] node names
+      # @yieldparam node [String] node name
+      # @yieldparam record [Proto::V1::NodeSchedule, nil] the current record, or nil when there is none
+      # @yieldreturn [Proto::V1::NodeSchedule, nil] the new record, or nil to delete it
+      # @return [Hash{String => Proto::V1::NodeSchedule}] records held afterwards, by node name
+      #: (String workflow, Array[String] nodes) { (String node, Proto::V1::NodeSchedule? record) -> Proto::V1::NodeSchedule? } -> Hash[String, Proto::V1::NodeSchedule]
+      def update_all(workflow, nodes, &) = raise NotImplementedError
+
+      # Replaces the record of one node with what the block returns.
+      #
+      # @param workflow [String] workflow name
+      # @param node [String] node name
+      # @yieldparam record [Proto::V1::NodeSchedule, nil] the current record, or nil when there is none
+      # @yieldreturn [Proto::V1::NodeSchedule, nil] the new record, or nil to delete it
+      # @return [Proto::V1::NodeSchedule, nil] the record held afterwards
+      #: (String workflow, String node) { (Proto::V1::NodeSchedule? record) -> Proto::V1::NodeSchedule? } -> Proto::V1::NodeSchedule?
+      def update(workflow, node, &block)
+        update_all(workflow, [node]) { |_node, record| block.call(record) }[node]
+      end
+
+      # Records due at or before a time and not leased to an orchestrator.
+      #
+      # @param workflow [String] workflow name
+      # @param now [Time] the time; a lease ending at or before it has expired
+      # @param limit [Integer, nil] records returned at most; all when nil
+      # @return [Array<Proto::V1::NodeSchedule>] the records, earliest due first
+      #: (String workflow, now: Time, ?limit: Integer?) -> Array[Proto::V1::NodeSchedule]
+      def list_due(workflow, now:, limit: nil) = raise NotImplementedError
+
+      # @param workflow [String] workflow name
+      # @return [Array<Proto::V1::NodeSchedule>] every record of the workflow, by node name
+      #: (String workflow) -> Array[Proto::V1::NodeSchedule]
+      def list(workflow) = raise NotImplementedError
+
+      # Releases resources held by the backend.
+      #
+      # @return [void]
+      #: () -> void
+      def close; end
+    end
+  end
+end