glazed 0.1.0

data/lib/glazed/state_backends/sqlite.rb

@@ -0,0 +1,299 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'fileutils'
+require 'securerandom'
+require 'tmpdir'
+require 'sqlite3'
+require 'glazed'
+require 'glazed/must'
+require 'glazed/proto'
+require 'glazed/runner'
+require 'glazed/state_backends/base'
+require 'glazed/step_context'
+require 'glazed/storages/base'
+require 'glazed/validation'
+
+module Glazed
+  module StateBackends
+    # SQLite database of node schedules. With a path, the file is the state
+    # and processes on the host share it through SQLite locking. With a
+    # storage, each process works on a private copy: it downloads the copy
+    # again whenever the storage holds a newer version, and uploads it after
+    # each change with a conditional write, redoing the change on the newer
+    # copy when another process uploaded first.
+    #
+    # @example Local file
+    #   Glazed::StateBackends::Sqlite.new(path: '/var/lib/glazed/state.sqlite3')
+    # @example Kept in S3
+    #   Glazed::StateBackends::Sqlite.new(storage: Glazed::Storages::S3.new(bucket: 'netconf'), key: 'glazed/state.sqlite3')
+    class Sqlite < Base
+      # Raised when the database was written by an incompatible glazed version.
+      class SchemaMismatch < Glazed::Error; end
+
+      # Version of the database layout.
+      SCHEMA_VERSION = 1 #: Integer
+      # Attempts at a change that keeps conflicting with other processes.
+      MAX_ATTEMPTS = 5 #: Integer
+      # Milliseconds to wait for another process holding the database lock.
+      BUSY_TIMEOUT_MS = 10_000 #: Integer
+
+      # @param path [String, nil] local database file, created when missing
+      # @param storage [Storages::Base, nil] storage holding the database, supporting conditional writes
+      # @param key [String, nil] key of the database in the storage
+      # @param listener [Proc] receives an event whenever a change is redone after a conflict
+      # @raise [ArgumentError] unless either path, or storage and key are given, or when the storage is not conditional
+      #: (?path: String?, ?storage: Storages::Base?, ?key: String?, ?listener: Runner::listener) -> void
+      def initialize(path: nil, storage: nil, key: nil, listener: ->(_event) {})
+        unless path.nil? ? storage && key : storage.nil? && key.nil?
+          raise ArgumentError, 'set either path, or storage and key'
+        end
+        raise ArgumentError, "#{storage.class.name} does not support conditional writes" if storage && !storage.conditional?
+
+        @storage = storage #: Storages::Base?
+        @key = key #: String?
+        @listener = listener #: Runner::listener
+        @tmpdir = (Dir.mktmpdir('glazed-state') unless path) #: String?
+        @path = path || ::File.join(Must.expect(@tmpdir), 'state.sqlite3') #: String
+        @db = nil #: SQLite3::Database?
+        @version = nil #: String?
+        @lock = Mutex.new #: Thread::Mutex
+        FileUtils.mkdir_p(::File.dirname(@path))
+      end
+
+      # Builds a state backend from {Proto::V1::SqliteStateBackend}.
+      #
+      # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::SqliteStateBackend}
+      # @param context [StepContext] configured storages, and where retries are reported
+      # @return [Sqlite] the state backend
+      # @raise [Must::Violation] when settings is another message
+      # @raise [StepContext::UnknownStorage] when the storage is not configured
+      # @raise [ArgumentError] when the storage does not support conditional writes
+      #: (Google::Protobuf::MessageExts settings, context: StepContext) -> Sqlite
+      def self.from_proto(settings, context:)
+        sqlite = Must.be_a(settings, Proto::V1::SqliteStateBackend)
+        return new(path: sqlite.path) unless sqlite.path.empty?
+
+        new(storage: context.storage(sqlite.storage), key: sqlite.key, listener: ->(event) { context.notify(**event) })
+      end
+
+      # @param workflow [String] workflow name
+      # @param nodes [Array<String>] node names
+      # @yieldparam node [String] node name
+      # @yieldparam record [Proto::V1::NodeSchedule, nil] the current record, or nil when there is none
+      # @yieldreturn [Proto::V1::NodeSchedule, nil] the new record, or nil to delete it
+      # @return [Hash{String => Proto::V1::NodeSchedule}] records held afterwards, by node name
+      # @raise [ArgumentError] when the block returns a record of another workflow or node
+      # @raise [InvalidMessage] when the block returns an invalid record
+      # @raise [Storages::Base::Conflict] when other processes keep changing the database
+      #: (String workflow, Array[String] nodes) { (String node, Proto::V1::NodeSchedule? record) -> Proto::V1::NodeSchedule? } -> Hash[String, Proto::V1::NodeSchedule]
+      def update_all(workflow, nodes, &block)
+        write do |db|
+          result = {} #: Hash[String, Proto::V1::NodeSchedule]
+          nodes.uniq.each do |node|
+            stored = db.get_first_value('select record from schedules where workflow = ? and node = ?', workflow, node) #: String?
+            updated = block.call(node, stored && Proto::V1::NodeSchedule.decode(stored))
+            unless updated
+              db.execute('delete from schedules where workflow = ? and node = ?', [workflow, node]) if stored
+              next
+            end
+
+            unless updated.workflow == workflow && updated.node == node
+              raise ArgumentError, "record of #{updated.workflow}/#{updated.node} returned for #{workflow}/#{node}"
+            end
+
+            Validation.validate!(updated)
+            encoded = Proto::V1::NodeSchedule.encode(updated)
+            save(db, updated, encoded) unless encoded == stored
+            result[node] = updated
+          end
+          result
+        end
+      end
+
+      # @param workflow [String] workflow name
+      # @param now [Time] the time; a lease ending at or before it has expired
+      # @param limit [Integer, nil] records returned at most; all when nil
+      # @return [Array<Proto::V1::NodeSchedule>] the records, earliest due first
+      #: (String workflow, now: Time, ?limit: Integer?) -> Array[Proto::V1::NodeSchedule]
+      def list_due(workflow, now:, limit: nil)
+        at = (now.to_r * 1000).floor
+        query(
+          'select record from schedules where workflow = ? and due_at_ms <= ? and lease_until_ms <= ? order by due_at_ms, node limit ?',
+          [workflow, at, at, limit || -1],
+        )
+      end
+
+      # @param workflow [String] workflow name
+      # @return [Array<Proto::V1::NodeSchedule>] every record of the workflow, by node name
+      #: (String workflow) -> Array[Proto::V1::NodeSchedule]
+      def list(workflow) = query('select record from schedules where workflow = ? order by node', [workflow])
+
+      # Closes the database and removes the private copy, if any.
+      #
+      # @return [void]
+      #: () -> void
+      def close
+        @lock.synchronize do
+          disconnect
+          tmpdir = @tmpdir
+          FileUtils.rm_rf(tmpdir) if tmpdir
+        end
+      end
+
+      # @param sql [String] select statement returning records
+      # @param binds [Array<String, Integer>] bound values
+      # @return [Array<Proto::V1::NodeSchedule>] the records
+      #: (String sql, Array[String | Integer] binds) -> Array[Proto::V1::NodeSchedule]
+      private def query(sql, binds)
+        @lock.synchronize do
+          rows = connection.execute(sql, binds) #: Array[[String]]
+          rows.map { |(record)| Proto::V1::NodeSchedule.decode(record) }
+        end
+      end
+
+      # Runs a change in a transaction, then uploads the database when it
+      # changed and is kept in a storage. A conflicting upload redoes the change.
+      #
+      # @yieldparam db [SQLite3::Database] the database, in a transaction
+      # @return [Object] what the block returns
+      # @raise [Storages::Base::Conflict] when the upload keeps conflicting
+      #: [T] () { (SQLite3::Database db) -> T } -> T
+      private def write(&block)
+        @lock.synchronize do
+          attempts = 0
+          begin
+            attempts += 1
+            db = connection
+            changes = db.total_changes
+            result = db.transaction(:immediate) { block.call(db) } #: T
+            upload if @storage && db.total_changes != changes
+            result
+          rescue Storages::Base::Conflict => e
+            raise if attempts >= MAX_ATTEMPTS
+
+            @listener.call({level: 'info', message: "state changed by another process, redoing the change: #{e.message}"})
+            sleep(rand * 0.05 * (2**attempts))
+            retry
+          end
+        end
+      end
+
+      # @param db [SQLite3::Database] the database, in a transaction
+      # @param record [Proto::V1::NodeSchedule] record to save
+      # @param encoded [String] the record encoded
+      # @return [void]
+      #: (SQLite3::Database db, Proto::V1::NodeSchedule record, String encoded) -> void
+      private def save(db, record, encoded)
+        db.execute(<<~SQL, [record.workflow, record.node, milliseconds(record.due_at), milliseconds(record.lease_until), SQLite3::Blob.new(encoded)])
+          insert into schedules (workflow, node, due_at_ms, lease_until_ms, record) values (?, ?, ?, ?, ?)
+          on conflict (workflow, node) do update set
+            due_at_ms = excluded.due_at_ms, lease_until_ms = excluded.lease_until_ms, record = excluded.record
+        SQL
+      end
+
+      # @param timestamp [Google::Protobuf::Timestamp, nil] the time
+      # @return [Integer] milliseconds since the epoch; 0 when unset
+      #: (Google::Protobuf::Timestamp? timestamp) -> Integer
+      private def milliseconds(timestamp)
+        timestamp ? (timestamp.seconds * 1000) + (timestamp.nanos / 1_000_000) : 0
+      end
+
+      # Opens the database, first replacing the private copy when the storage
+      # holds a version this process has not seen.
+      #
+      # @return [SQLite3::Database] the database
+      # @raise [SchemaMismatch] when the database has an incompatible layout
+      #: () -> SQLite3::Database
+      private def connection
+        storage = @storage
+        key = @key
+        db = @db
+        return db || connect unless storage && key
+        return db if db && storage.version_of(key) == @version
+
+        disconnect
+        latest = storage.read_versioned(key)
+        if latest
+          temporary = "#{@path}.#{SecureRandom.hex(8)}.tmp"
+          ::File.binwrite(temporary, latest.content)
+          ::File.rename(temporary, @path)
+        end
+        @version = latest&.version
+        connect
+      end
+
+      # Uploads the private copy, expecting the storage to still hold the version it was made from.
+      #
+      # @return [void]
+      # @raise [Storages::Base::Conflict] when the storage holds another version
+      #: () -> void
+      private def upload
+        succeeded = false
+        @version = Must.expect(@storage).write_if(key: Must.expect(@key), content: ::File.binread(@path), expected_version: @version)
+        succeeded = true
+      ensure
+        # The private copy holds a change the storage lacks, so it is fetched again.
+        disconnect unless succeeded
+      end
+
+      # @return [SQLite3::Database] the database, with its schema ensured
+      # @raise [SchemaMismatch] when the database has an incompatible layout
+      #: () -> SQLite3::Database
+      private def connect
+        db = SQLite3::Database.new(@path, strict: true)
+        db.busy_timeout = BUSY_TIMEOUT_MS
+        ensure_schema(db)
+        @db = db
+      rescue SchemaMismatch
+        db&.close
+        raise
+      end
+
+      # Closes the database, removing the private copy so it is fetched again.
+      #
+      # @return [void]
+      #: () -> void
+      private def disconnect
+        @db&.close
+        @db = nil
+        return unless @storage
+
+        ::File.unlink(@path) if ::File.exist?(@path)
+        @version = nil
+      end
+
+      # @param db [SQLite3::Database] the database
+      # @return [void]
+      # @raise [SchemaMismatch] when the database has an incompatible layout
+      #: (SQLite3::Database db) -> void
+      private def ensure_schema(db)
+        db.transaction(:immediate) do
+          db.execute('create table if not exists glazed_schema (version integer not null) strict')
+          version = db.get_first_value('select version from glazed_schema') #: Integer?
+          case version
+          when SCHEMA_VERSION
+            nil
+          when nil
+            db.execute(<<~SQL)
+              create table schedules (
+                workflow text not null,
+                node text not null,
+                due_at_ms integer not null,
+                lease_until_ms integer not null,
+                record blob not null,
+                primary key (workflow, node)
+              ) strict
+            SQL
+            db.execute('create index schedules_due on schedules (workflow, due_at_ms)')
+            db.execute('insert into glazed_schema (version) values (?)', [SCHEMA_VERSION])
+          else
+            raise SchemaMismatch, "#{@path} has state schema version #{version}, while this glazed uses #{SCHEMA_VERSION}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/glazed/state_backends.rb

@@ -0,0 +1,75 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'glazed'
+require 'glazed/finder'
+require 'glazed/must'
+require 'glazed/plugin_registry'
+require 'glazed/proto'
+require 'glazed/state_backends/base'
+require 'glazed/step_context'
+require 'glazed/validation'
+
+module Glazed
+  # Namespace of state backend implementations, looked up by name.
+  module StateBackends
+    # Raised when a state backend config names no backend.
+    class MissingBackend < Glazed::Error; end
+
+    # Finds a state backend class by snake_case name.
+    #
+    # @param name [String, Symbol] state backend name, such as "sqlite"
+    # @return [Class<Base>] the state backend class
+    # @raise [Finder::NotFound] when there is none, for example when the gem providing it is not installed
+    # @raise [Must::Violation] when the name resolves to something other than a state backend
+    #: (String | Symbol name) -> singleton(Base)
+    def self.find(name) = state_backend_class(Finder.find(self, 'glazed/state_backends', name))
+
+    # State backends implemented by plugins, by settings message type.
+    PLUGINS = PluginRegistry.new('state backend') #: PluginRegistry
+
+    # Registers a plugin state backend, selected by `plugin` configs whose
+    # settings are of the given type.
+    #
+    # @param settings_class [Class] generated message class of the state backend settings
+    # @param implementation [Class<Base>] state backend built with {Base.from_proto}
+    # @return [void]
+    #: (singleton(Google::Protobuf::AbstractMessage) settings_class, singleton(Base) implementation) -> void
+    def self.register(settings_class, implementation)
+      PLUGINS.register(settings_class, implementation)
+    end
+
+    # Builds the state backend a config selects.
+    #
+    # @param config [Proto::V1::StateBackendConfig] state backend settings
+    # @param context [StepContext] configured storages, and where progress is reported
+    # @return [Base] the state backend
+    # @raise [InvalidMessage] when the config breaks the rules of its schema
+    # @raise [Finder::NotFound] when the backend is not available
+    # @raise [PluginRegistry::UnknownPlugin] when no plugin is registered for the settings type
+    #: (Proto::V1::StateBackendConfig config, context: StepContext) -> Base
+    def self.build(config, context:)
+      Validation.validate!(config)
+      backend = config.backend or raise MissingBackend, 'state backend config sets no backend'
+      if (packed = config.plugin)
+        implementation, settings = PLUGINS.resolve(packed)
+        return state_backend_class(implementation).from_proto(settings, context:)
+      end
+
+      settings = Must.be_a(config[backend.to_s], Google::Protobuf::MessageExts)
+      find(backend).from_proto(settings, context:)
+    end
+
+    # @param implementation [Module] registered or found implementation
+    # @return [Class<Base>] the implementation as a state backend class
+    # @raise [Must::Violation] when it is not a state backend
+    #: (Module[untyped] implementation) -> singleton(Base)
+    private_class_method def self.state_backend_class(implementation)
+      return implementation if implementation.is_a?(Class) && implementation < Base
+
+      raise Must::Violation, "#{implementation} is not a state backend"
+    end
+  end
+end

data/lib/glazed/step_context.rb

@@ -0,0 +1,65 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'google/protobuf'
+require 'glazed'
+require 'glazed/proto'
+require 'glazed/storages'
+
+module Glazed
+  # Named storages shared by the steps of a workflow, and where steps report progress.
+  class StepContext
+    # Raised when a step names a storage that is not configured.
+    class UnknownStorage < Glazed::Error; end
+
+    # @param storages [Hash{String => Storages::Base}] storages by name
+    # @param listener [Proc] receives progress events from steps, possibly from several threads at once
+    #: (storages: Hash[String, Storages::Base], ?listener: Runner::listener) -> void
+    def initialize(storages:, listener: ->(_event) {})
+      @storages = storages #: Hash[String, Storages::Base]
+      @listener = listener #: Runner::listener
+    end
+
+    # Builds every configured storage.
+    #
+    # @param storages [Google::Protobuf::Map] storage configs by name, from {Proto::V1::Config#storages}
+    # @param listener [Proc] receives progress events from steps, possibly from several threads at once
+    # @return [StepContext] the context
+    # @raise [Storages::MissingBackend] when a storage config sets no backend
+    #: (Google::Protobuf::Map[String, Proto::V1::StorageConfig] storages, ?listener: Runner::listener) -> StepContext
+    def self.from_proto(storages, listener: ->(_event) {})
+      # Map#to_h would turn the messages into hashes.
+      built = {} #: Hash[String, Storages::Base]
+      storages.each { |name, config| built[name] = Storages.build(config) }
+      new(storages: built, listener:)
+    end
+
+    # Reports progress of a step.
+    #
+    # @param event [Hash{Symbol => String, Integer, Boolean, nil}] event fields
+    # @return [void]
+    #: (**Runner::event_value event) -> void
+    def notify(**event) = @listener.call(event)
+
+    # @param name [String] storage name
+    # @return [Storages::Base] the storage
+    # @raise [UnknownStorage] when no storage has that name
+    #: (String name) -> Storages::Base
+    def storage(name)
+      @storages.fetch(name) { raise UnknownStorage, "no storage named #{name.inspect}" }
+    end
+
+    # Reads an artifact's content from its storage.
+    #
+    # @param artifact [Proto::V1::Artifact] the artifact
+    # @return [String] its content
+    # @raise [UnknownStorage] when the artifact's storage is not configured
+    #: (Proto::V1::Artifact artifact) -> String
+    def read(artifact)
+      pointer = artifact.pointer or raise ArgumentError, "artifact in #{artifact.storage} has no pointer"
+      storage(artifact.storage).read(pointer)
+    end
+  end
+end

data/lib/glazed/steps/base.rb

@@ -0,0 +1,89 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'google/protobuf'
+require 'glazed'
+require 'glazed/proto'
+require 'glazed/storages/base'
+
+module Glazed
+  module Steps
+    # Base of workflow steps. Subclass {NodeStep} or {BatchStep}.
+    class Base
+      # Raised when a state lacks an artifact the step reads.
+      class MissingArtifact < Glazed::Error; end
+
+      # Builds a step from its settings message.
+      #
+      # @param name [String] step name, unique within the workflow
+      # @param settings [Google::Protobuf::MessageExts] action settings from {Proto::V1::Step}
+      # @param context [StepContext] named storages
+      # @return [Base] the step
+      #: (String name, Google::Protobuf::MessageExts settings, context: StepContext) -> Base
+      def self.from_proto(name, settings, context:) = raise NotImplementedError
+
+      # @param name [String] step name, unique within the workflow
+      #: (name: String) -> void
+      def initialize(name:)
+        @name = name #: String
+      end
+
+      # @return [String] step name
+      #: String
+      attr_reader :name
+
+      # @return [Array<String>] names of the artifacts the step reads
+      #: () -> Array[String]
+      def inputs = []
+
+      # @return [Array<String>] names of the artifacts the step produces
+      #: () -> Array[String]
+      def outputs = []
+
+      # @param state [Proto::V1::NodeState] state to read from
+      # @param artifact [String] artifact name
+      # @return [Proto::V1::Artifact] the artifact
+      # @raise [MissingArtifact] when the state lacks it
+      #: (Proto::V1::NodeState state, String artifact) -> Proto::V1::Artifact
+      private def artifact(state, artifact)
+        state.artifacts[artifact] or raise MissingArtifact, "#{name}: #{state.node&.name} has no artifact #{artifact.inspect}"
+      end
+
+      # @param storage_name [String] name of the storage written to
+      # @param result [Storages::Base::WriteResult] outcome of the write
+      # @return [Proto::V1::Artifact] artifact naming the stored content and whether it changed
+      #: (String storage_name, Storages::Base::WriteResult result) -> Proto::V1::Artifact
+      private def stored(storage_name, result)
+        Proto::V1::Artifact.new(storage: storage_name, pointer: result.pointer, changed: result.changed?)
+      end
+
+      # @param state [Proto::V1::NodeState] state to extend
+      # @param artifacts [Hash{String => Proto::V1::Artifact}] artifacts to add
+      # @return [Proto::V1::NodeState] a copy of the state with the artifacts
+      #: (Proto::V1::NodeState state, Hash[String, Proto::V1::Artifact] artifacts) -> Proto::V1::NodeState
+      private def with_artifacts(state, artifacts)
+        copy = Proto::V1::NodeState.decode(Proto::V1::NodeState.encode(state))
+        artifacts.each { |key, value| copy.artifacts[key] = value }
+        copy
+      end
+    end
+
+    # Step run for each node independently.
+    class NodeStep < Base
+      # @param state [Proto::V1::NodeState] the node and its artifacts
+      # @return [Proto::V1::NodeState] the state with the step's outputs added
+      #: (Proto::V1::NodeState state) -> Proto::V1::NodeState
+      def call(state) = raise NotImplementedError
+    end
+
+    # Step run once with every node that completed the steps before it.
+    class BatchStep < Base
+      # @param batch [Proto::V1::BatchState] node states
+      # @return [Proto::V1::BatchState] the states with the step's outputs added
+      #: (Proto::V1::BatchState batch) -> Proto::V1::BatchState
+      def call(batch) = raise NotImplementedError
+    end
+  end
+end

data/lib/glazed/steps/batch_shell.rb

@@ -0,0 +1,57 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'glazed/must'
+require 'glazed/proto'
+require 'glazed/proto_json'
+require 'glazed/shell_command'
+require 'glazed/steps'
+
+module Glazed
+  module Steps
+    # Batch: runs a command with /bin/sh -c once, with the
+    # {Proto::V1::BatchState} of every node that got this far as ProtoJSON on
+    # stdin, for hooks such as notifications.
+    #
+    # @example Step definition
+    #   {"name": "notify", "batch_shell": {"run": "bin/notify-changes"}}
+    class BatchShell < BatchStep
+      # Seconds a command may run when the definition sets none.
+      DEFAULT_TIMEOUT = 300 #: Integer
+
+      # @param name [String] step name
+      # @param run [String] command line
+      # @param timeout [Numeric] seconds before the command is killed
+      #: (name: String, run: String, ?timeout: Numeric) -> void
+      def initialize(name:, run:, timeout: DEFAULT_TIMEOUT)
+        super(name:)
+        @run = run #: String
+        @timeout = timeout #: Numeric
+      end
+
+      # Builds the step from {Proto::V1::BatchShellStep}.
+      #
+      # @param name [String] step name
+      # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::BatchShellStep}
+      # @param context [StepContext] named storages, unused
+      # @return [BatchShell] the step
+      #: (String name, Google::Protobuf::MessageExts settings, context: StepContext) -> BatchShell
+      def self.from_proto(name, settings, context:)
+        shell = Must.be_a(settings, Proto::V1::BatchShellStep)
+        new(name:, run: shell.run, timeout: shell.timeout_seconds.zero? ? DEFAULT_TIMEOUT : shell.timeout_seconds)
+      end
+
+      # @param batch [Proto::V1::BatchState] node states
+      # @return [Proto::V1::BatchState] the same states
+      # @raise [ShellCommand::Failed] when the command exits non-zero
+      # @raise [TimeoutError] when the command does not finish in time
+      #: (Proto::V1::BatchState batch) -> Proto::V1::BatchState
+      def call(batch)
+        ShellCommand.new(run: @run, env: {'GLAZED_STEP' => name}, stdin: ProtoJson.dump(batch), timeout: @timeout).call
+        batch
+      end
+    end
+  end
+end

data/lib/glazed/steps/fetch.rb

@@ -0,0 +1,67 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'glazed/cli_session'
+require 'glazed/collector'
+require 'glazed/must'
+require 'glazed/proto'
+require 'glazed/steps'
+require 'glazed/storages'
+
+module Glazed
+  module Steps
+    # Per node: collects the node's configuration and stores it as an artifact.
+    #
+    # @example Step definition
+    #   {"name": "fetch", "fetch": {"output": "raw", "storage": "vault"}}
+    class Fetch < NodeStep
+      # @param name [String] step name
+      # @param output [String] artifact to produce
+      # @param storage_name [String] storage to write to, by name
+      # @param context [StepContext] named storages, and where progress is reported
+      # @param timeout [Numeric] seconds to wait for each expected device output
+      # @raise [StepContext::UnknownStorage] when the storage is not configured
+      #: (name: String, output: String, storage_name: String, context: StepContext, ?timeout: Numeric) -> void
+      def initialize(name:, output:, storage_name:, context:, timeout: CliSession::DEFAULT_TIMEOUT)
+        super(name:)
+        @output = output #: String
+        @storage_name = storage_name #: String
+        @context = context #: StepContext
+        @storage = context.storage(storage_name) #: Storages::Base
+        @timeout = timeout #: Numeric
+      end
+
+      # Builds the step from {Proto::V1::FetchStep}.
+      #
+      # @param name [String] step name
+      # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::FetchStep}
+      # @param context [StepContext] named storages
+      # @return [Fetch] the step
+      #: (String name, Google::Protobuf::MessageExts settings, context: StepContext) -> Fetch
+      def self.from_proto(name, settings, context:)
+        fetch = Must.be_a(settings, Proto::V1::FetchStep)
+        timeout = fetch.timeout_seconds.zero? ? CliSession::DEFAULT_TIMEOUT : fetch.timeout_seconds
+        new(name:, output: fetch.output, storage_name: fetch.storage, context:, timeout:)
+      end
+
+      # @return [Array<String>] the produced artifact
+      #: () -> Array[String]
+      def outputs = [@output]
+
+      # @param state [Proto::V1::NodeState] the node
+      # @return [Proto::V1::NodeState] the state with the collected configuration
+      # @raise [ArgumentError] when the state has no node
+      # @raise [Glazed::Error] when collecting or storing fails
+      #: (Proto::V1::NodeState state) -> Proto::V1::NodeState
+      def call(state)
+        node = state.node or raise ArgumentError, "#{name}: state has no node"
+        progress = ->(message) { @context.notify(level: 'info', step: name, node: node.name, message:) } #: ^(String) -> void
+        config = Collector.new(node:, timeout: @timeout, progress:).call
+        result = @storage.write(key: Storages.key_for(node), content: config, message: "#{node.name}: #{name}")
+        with_artifacts(state, @output => stored(@storage_name, result))
+      end
+    end
+  end
+end