glazed 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f09d16d28a4ecb996fa05ba53cc4af10d5cbe45b65c09f3830f88326a6e200c5
-  data.tar.gz: 7c40270ad31553fac8a681ba5369c1596cfaed4a8c09cf66036115def90bd4d9
+  metadata.gz: e764c4f949d82239b29b22b0d38b300f260e31818bf41adc5fb47ad6c091468a
+  data.tar.gz: 5788a04e0b8da61d1c46e436520ad9e3f78641310b4e0f66c210c524d0041e64
 SHA512:
-  metadata.gz: d12cde4376d07ae82221202ebab2ec22a473b85dcdae0e2a5352d2f82235ed8e059a7a66fc7266d1d6d5e14eac5d68b2fdc05a34b22991aac18111d1620bd21e
-  data.tar.gz: b0e2f351f6ff580fd4b639162088ad4bfd3d947d8591805453129703ed227569d94c605124d43d760824d650c318450b6ab93766886c5e1fc2cd859b4eec1373
+  metadata.gz: f456a6b2116655636c1c19e3c8253131aecc266e2d72201ecc6d531faa7ee2e8d64d4ca92147bed91f2153a7e6810c5db8268ca31c257d2cbb08db1160731a02
+  data.tar.gz: 9d4d631c639884e667c90b718476d1c99a2d37c49085bdf9c48eb7c9a4f2d96626b2681cd5cd12a62abc20e1169b5371ce313d29d6ddffcfea430e8f65cd2776

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## [0.3.0] - 2026-10-17
+
+### Breaking changes
+
+- The `wlc` dialect is renamed `aireos` (`Glazed::Dialects::Aireos`), as Catalyst 9800 controllers run IOS XE
+
+### Enhancements
+
+- `aruba_instant` dialect for Aruba Instant (IAP and virtual controller), ported from oxidized's arubainstant model, with secret redaction for management users, SNMP, RADIUS/TACACS keys and passphrases
+- `eos` dialect for Arista EOS, ported from oxidized, redacting hashed and typed secrets, SNMP, AAA, routing, key chain and TerminAttr keys
+- `routeros` dialect for MikroTik RouterOS (SSH exec channel or Telnet), ported from Oxidized, redacting passwords, pre-shared keys, private keys, RADIUS and IPsec secrets, and SNMP community names. Param `routeros_hide_sensitive` runs `/export hide-sensitive`
+- `Glazed::Coordinator` takes `waiter_alive:`, telling whether the execution holding the waiter record still runs, so `arm` takes over from one that ended without giving its record up
+- `aireos` redacts the encrypted secrets AireOS prints in its configuration, along with mgmtuser, netuser, AP mgmtuser and SNMP passwords, and SNMP community names wherever they appear
+- `Glazed::NodeSession` connects to a node, logs in and runs the dialect's post-login and pre-logout hooks, for callers that keep an interactive session open. `exec: false` keeps commands on the shell where the dialect would use exec channels, and `interactions:` adds responders after the dialect's own. `Collector` is built on it
+
 ## [0.2.0] - 2026-10-15
 
 ### Breaking changes
@@ -8,6 +23,8 @@
 
 ### Enhancements
 
+- New `routeros` dialect for MikroTik RouterOS (SSH exec channel or Telnet), ported from Oxidized, redacting passwords, pre-shared keys, private keys, RADIUS and IPsec secrets, and SNMP community names. Param `routeros_hide_sensitive` runs `/export hide-sensitive`
+
 - Redaction hides additional secrets given as secret references, as literal text or regular expressions and optionally split by a delimiter, in a `redact` step or in the top-level `redaction` shared by all of them
 - Storages: `subdirectory` for `git` and `github` places files in a directory of the repository instead of its root
 - `glazed signal --all`, and `all_nodes` and `workflows` on `glazed.v1.Signal`, signal every node of chosen workflows, for scheduled runs
@@ -23,6 +40,8 @@ Initial release.
 
 ### Enhancements
 
+- New `routeros` dialect for MikroTik RouterOS (SSH exec channel or Telnet), ported from Oxidized, redacting passwords, pre-shared keys, private keys, RADIUS and IPsec secrets, and SNMP community names. Param `routeros_hide_sensitive` runs `/export hide-sensitive`
+
 - Collect configuration over SSH or Telnet, with dialects for Cisco IOS/IOS XE, Cisco WLC (AireOS), Juniper Junos, NEC UNIVERGE IX and Allied Telesis AlliedWare Plus, ported from Oxidized
 - Class-based dialect DSL with inheritance ([docs/dialect-api.md](../docs/dialect-api.md))
 - Redaction of the secrets a dialect locates: replaced with `[~REDACTED~]`, or with `[~REDACTED:dgst=…~]` carrying a keyed HMAC so changed secrets still show in diffs

data/NOTICE

@@ -11,6 +11,6 @@ Version 2.0:
 
 Derived from Oxidized as of commit d7f51c2, and modified for glazed:
 
-- lib/glazed/dialects/{awplus,ios,junos,nec_ix,wlc}.rb
+- lib/glazed/dialects/{aireos,aruba_instant,awplus,eos,ios,junos,nec_ix,routeros}.rb
   Prompts, command sequences, output cleanup and secret patterns ported
-  from lib/oxidized/model/{awplus,ios,junos,necix,aireos}.rb.
+  from lib/oxidized/model/{awplus,ios,junos,necix,aireos,eos,arubainstant,routeros}.rb.

data/lib/glazed/cli_session.rb

@@ -22,7 +22,7 @@ module Glazed
   #   session = Glazed::CliSession.new(transport:, prompt: /^\S+#\s?$/)
   #   session.command('show version').output
   class CliSession
-    # Default seconds to wait for expected output.
+    # Default seconds the device may stay silent while expected output is awaited.
     DEFAULT_TIMEOUT = 60 #: Integer
     # Longest single wait on the transport, so a deadline is checked regularly.
     READ_SLICE = 1.0 #: Float
@@ -32,7 +32,7 @@ module Glazed
     # @param data_filters [Array<Proc>] filters applied to each received fragment, in order
     # @param interactions [Array<Dialect::Interaction>] responders to patterns in the buffer
     # @param exec [Boolean] run {#command} through {Transports::Base#exec} when supported
-    # @param timeout [Numeric] default seconds to wait for expected output
+    # @param timeout [Numeric] default seconds the device may stay silent while output is awaited
     # @param progress [Proc] receives a message whenever the session starts waiting for the device
     #: (transport: Transports::Base, prompt: Regexp, ?data_filters: Array[Dialect::data_filter], ?interactions: Array[Dialect::Interaction], ?exec: bool, ?timeout: Numeric, ?progress: ^(String) -> void) -> void
     def initialize(transport:, prompt:, data_filters: [], interactions: [], exec: false, timeout: DEFAULT_TIMEOUT, progress: ->(_message) {})
@@ -72,9 +72,9 @@ module Glazed
     # the end of the match.
     #
     # @param patterns [Array<Regexp>] patterns tried in order
-    # @param timeout [Numeric] seconds to wait
+    # @param timeout [Numeric] seconds the device may stay silent
     # @return [MatchData] the first match
-    # @raise [TimeoutError] when nothing matches in time
+    # @raise [TimeoutError] when the device stays silent for timeout seconds before a match
     # @raise [ConnectionClosed] when the device closes the connection first
     #: (*Regexp patterns, ?timeout: Numeric) -> MatchData
     def expect(*patterns, timeout: self.timeout)
@@ -87,9 +87,9 @@ module Glazed
     # Runs a command and waits for the prompt.
     #
     # @param line [String] command line, without newline
-    # @param timeout [Numeric] seconds to wait for the prompt
+    # @param timeout [Numeric] seconds the device may stay silent before the prompt
     # @return [CommandResult] output without the echoed command and the prompt
-    # @raise [TimeoutError] when the prompt does not appear in time
+    # @raise [TimeoutError] when the device stays silent for timeout seconds before the prompt
     # @raise [ConnectionClosed] when the device closes the connection first
     #: (String line, ?timeout: Numeric) -> CommandResult
     def command(line, timeout: self.timeout)
@@ -122,11 +122,14 @@ module Glazed
       discarded
     end
 
+    # Waits for a match while the device keeps sending. The timeout restarts
+    # whenever data arrives, so slow but steady output does not time out.
+    #
     # @param description [String] what is awaited, for error messages
-    # @param timeout [Numeric] seconds to wait
+    # @param timeout [Numeric] seconds the device may stay silent
     # @yieldreturn [MatchData, nil] match against the current buffer, or nil to keep waiting
     # @return [MatchData] the first non-nil match
-    # @raise [TimeoutError] when nothing matches in time
+    # @raise [TimeoutError] when the device stays silent for timeout seconds before a match
     # @raise [ConnectionClosed] when the device closes the connection first
     #: (String description, timeout: Numeric) { () -> MatchData? } -> MatchData
     private def wait_until(description, timeout:, &matcher)
@@ -137,12 +140,14 @@ module Glazed
         return match if match
 
         remaining = deadline - now
-        raise TimeoutError, "timed out after #{timeout}s waiting for #{description}; buffer tail: #{(buffer[-200..] || buffer).dump}" unless remaining.positive?
+        raise TimeoutError, "no output for #{timeout}s while waiting for #{description}; buffer tail: #{(buffer[-200..] || buffer).dump}" unless remaining.positive?
 
         data = transport.read(timeout: [remaining, READ_SLICE].min)
         raise ConnectionClosed, "connection closed while waiting for #{description}" if data.nil?
+        next if data.empty?
 
-        feed(data) unless data.empty?
+        deadline = now + timeout.to_f
+        feed(data)
       end
     end
 

data/lib/glazed/collector.rb

@@ -5,8 +5,7 @@
 
 require 'glazed'
 require 'glazed/cli_session'
-require 'glazed/dialect/run'
-require 'glazed/dialects'
+require 'glazed/node_session'
 require 'glazed/proto'
 require 'glazed/secrets_resolver'
 require 'glazed/transports'
@@ -18,9 +17,6 @@ module Glazed
   # @example
   #   config = Glazed::Collector.new(node:).call
   class Collector
-    # Transport used when a node names none.
-    DEFAULT_TRANSPORT = 'ssh' #: String
-
     # @param node [Proto::V1::Node] node to collect from
     # @param transport [Transports::Base, nil] transport to use instead of building one from the node
     # @param timeout [Numeric] seconds to wait for each expected output
@@ -28,22 +24,16 @@ module Glazed
     # @param secrets [SecretsResolver] reads the node's password and secrets
     #: (node: Proto::V1::Node, ?transport: Transports::Base?, ?timeout: Numeric, ?progress: ^(String) -> void, ?secrets: SecretsResolver) -> void
     def initialize(node:, transport: nil, timeout: CliSession::DEFAULT_TIMEOUT, progress: ->(_message) {}, secrets: SecretsResolver.new)
-      @secrets = secrets #: SecretsResolver
-      @node = node #: Proto::V1::Node
-      @dialect = Dialects.find(node.dialect) #: singleton(Dialects::Base)
-      @kind = (node.transport.empty? ? DEFAULT_TRANSPORT : node.transport) #: String
-      @transport = transport || build_transport #: Transports::Base
-      @config = @dialect.communication_config(@kind.to_sym) #: Dialect::CommunicationConfig
-      @timeout = timeout #: Numeric
-      @progress = progress #: ^(String) -> void
+      @session = NodeSession.new(node:, transport:, timeout:, progress:, secrets:) #: NodeSession
     end
 
     # @return [Proto::V1::Node] node to collect from
-    #: Proto::V1::Node
-    attr_reader :node
+    #: () -> Proto::V1::Node
+    def node = @session.node
+
     # @return [Class<Dialects::Base>] dialect of the node
-    #: singleton(Dialects::Base)
-    attr_reader :dialect
+    #: () -> singleton(Dialects::Base)
+    def dialect = @session.dialect
 
     # Collects the configuration. The connection is closed even on failure.
     #
@@ -53,65 +43,16 @@ module Glazed
     # @raise [InvalidEncoding] when the configuration is not valid UTF-8
     #: () -> String
     def call
-      session = CliSession.new(
-        transport: @transport,
-        prompt: dialect.prompt_pattern,
-        data_filters: dialect.data_filters,
-        interactions: dialect.interactions,
-        exec: @config.exec,
-        timeout: @timeout,
-        progress: @progress,
-      )
-      run = Dialect::Run.new(cli: session, node:, dialect:, secrets: @secrets)
-      @transport.pty_options = @config.pty_options
-      @progress.call("connecting to #{@transport.host} over #{@kind}")
-      @transport.open
+      @session.open
       begin
-        login(session) unless exec_only?
-        @config.post_login_hooks.each { it.call(run) }
-        output = dialect.runbook_for(:get).execute(run).dup.force_encoding(Encoding::UTF_8)
+        output = dialect.runbook_for(:get).execute(@session.run).dup.force_encoding(Encoding::UTF_8)
         raise InvalidEncoding, "#{node.name}: collected configuration is not valid UTF-8" unless output.valid_encoding?
 
-        @config.pre_logout_hooks.each { it.call(run) }
+        @session.logout
         output
       ensure
-        @transport.close
-      end
-    end
-
-    # @return [Boolean] whether every command runs on exec channels, so the shell is never used
-    #: () -> bool
-    private def exec_only? = @config.exec && @transport.exec?
-
-    # Answers login prompts the dialect declares, then waits for the first prompt.
-    #
-    # @param session [CliSession] session to log in through
-    # @return [void]
-    #: (CliSession session) -> void
-    private def login(session)
-      if (username_prompt = @config.username_prompt)
-        session.expect(username_prompt)
-        session.write("#{node.username}\n")
-      end
-      if (password_prompt = @config.password_prompt)
-        session.expect(password_prompt)
-        session.write("#{@secrets.maybe(node.password)&.value}\n")
+        @session.close
       end
-      session.expect(session.prompt)
-    end
-
-    # @return [Transports::Base] transport built from the node
-    # @raise [Finder::NotFound] when the transport is unknown
-    #: () -> Transports::Base
-    private def build_transport
-      Transports.find(@kind).new(
-        host: node.host.empty? ? node.name : node.host,
-        port: node.port.zero? ? nil : node.port,
-        username: node.username,
-        password: @secrets.maybe(node.password),
-        transport_options: node.transport_options,
-        secrets: @secrets,
-      )
     end
   end
 end

data/lib/glazed/command/base.rb

@@ -85,10 +85,20 @@ module Glazed
         StateBackends.build(Must.because(orchestration.state) { 'validation requires orchestrator.state' }, context:)
       end
 
+      # dry-cli passes integer options as strings.
+      #
+      # @param option [String, nil] the --jobs option
       # @param config [Proto::V1::Config] the config
       # @return [Integer] nodes processed in parallel
-      #: (Proto::V1::Config config) -> Integer
-      private def default_jobs(config) = config.jobs.zero? ? DEFAULT_JOBS : config.jobs
+      # @raise [Glazed::Error] when --jobs is not a positive integer
+      #: (String? option, Proto::V1::Config config) -> Integer
+      private def resolve_jobs(option, config)
+        return config.jobs.zero? ? DEFAULT_JOBS : config.jobs unless option
+
+        raise Glazed::Error, "--jobs must be a positive integer, got #{option.inspect}" unless option.match?(/\A[1-9]\d*\z/)
+
+        option.to_i
+      end
 
       # Reads the config with {Glazed::Config.load}.
       #

data/lib/glazed/command/orchestrate.rb

@@ -26,12 +26,12 @@ module Glazed
       # Runs the orchestrator until a signal stops it.
       #
       # @param config [String, nil] path of the config file; {Base::CONFIG_FILE_ENV} when nil
-      # @param jobs [Integer, nil] parallelism overriding the config
+      # @param jobs [String, nil] parallelism overriding the config
       # @param _options [Hash] other options dry-cli passes, unused
       # @return [void]
       # @raise [Glazed::Error, Google::Protobuf::ParseError] when the config or inventory is invalid, or the state backend fails
       # @raise [Interrupt] when a second TERM or INT interrupts a batch
-      #: (?config: String?, ?jobs: Integer?, **untyped _options) -> void
+      #: (?config: String?, ?jobs: String?, **untyped _options) -> void
       def call(config: nil, jobs: nil, **_options)
         config = config_path(config, 'orchestrate')
 
@@ -43,7 +43,7 @@ module Glazed
         backend = state_backend(orchestration, context)
         orchestrator = Orchestrator.new(
           settings: orchestration, workflows:, backend:, inventory: -> { Sources.nodes(settings.inventory.to_a) },
-          jobs: jobs || default_jobs(settings), listener:,
+          jobs: resolve_jobs(jobs, settings), listener:,
         )
         with_traps(orchestrator) { orchestrator.run }
       ensure

data/lib/glazed/command/run.rb

@@ -27,11 +27,11 @@ module Glazed
       # @param config [String, nil] path of the config file; {Base::CONFIG_FILE_ENV} when nil
       # @param workflow [String, nil] workflow name; the only one when nil
       # @param only [Array<String>, nil] node names to run for; all when nil
-      # @param jobs [Integer, nil] parallelism overriding the config
+      # @param jobs [String, nil] parallelism overriding the config
       # @param _options [Hash] other options dry-cli passes, unused
       # @return [void]
       # @raise [Glazed::Error, Google::Protobuf::ParseError] when the config, inventory or a batch step fails
-      #: (?config: String?, ?workflow: String?, ?only: Array[String]?, ?jobs: Integer?, **untyped _options) -> void
+      #: (?config: String?, ?workflow: String?, ?only: Array[String]?, ?jobs: String?, **untyped _options) -> void
       def call(config: nil, workflow: nil, only: nil, jobs: nil, **_options)
         config = config_path(config, 'run')
 
@@ -41,8 +41,7 @@ module Glazed
         steps = Workflow.from_proto(select_workflow(settings, workflow), context:)
         nodes = select_nodes(Sources.nodes(settings.inventory.to_a), only || [])
 
-        jobs ||= default_jobs(settings)
-        result = Runner.new(workflow: steps, jobs:, listener:).call(nodes)
+        result = Runner.new(workflow: steps, jobs: resolve_jobs(jobs, settings), listener:).call(nodes)
         out.puts "completed #{result.states.size} of #{nodes.size} nodes, #{result.changed.size} changed, #{result.failures.size} failed"
         exit 1 unless result.failures.empty?
       end

data/lib/glazed/coordinator.rb

@@ -36,15 +36,17 @@ module Glazed
     # @param clock [Proc] returns the current time
     # @param random [Random] source of jitter
     # @param owner [String] identifies this process in leases
-    #: (settings: Proto::V1::Orchestrator, backend: StateBackends::Base, ?listener: Runner::listener, ?clock: ^() -> Time, ?random: Random, ?owner: String) -> void
+    # @param waiter_alive [Proc] tells whether the execution of a waiter's token still runs; see {#arm}
+    #: (settings: Proto::V1::Orchestrator, backend: StateBackends::Base, ?listener: Runner::listener, ?clock: ^() -> Time, ?random: Random, ?owner: String, ?waiter_alive: ^(String) -> bool) -> void
     def initialize(settings:, backend:, listener: ->(_event) {}, clock: -> { Time.now }, random: Random.new,
-                   owner: "#{Socket.gethostname}:#{Process.pid}:#{SecureRandom.hex(4)}")
+                   owner: "#{Socket.gethostname}:#{Process.pid}:#{SecureRandom.hex(4)}", waiter_alive: ->(_token) { true })
       @settings = settings #: Proto::V1::Orchestrator
       @backend = backend #: StateBackends::Base
       @listener = listener #: Runner::listener
       @clock = clock #: ^() -> Time
       @random = random #: Random
       @owner = owner #: String
+      @waiter_alive = waiter_alive #: ^(String) -> bool
     end
 
     # @return [Proto::V1::Orchestrator] orchestrator settings
@@ -202,7 +204,8 @@ module Glazed
 
     # Decides whether a coordinator execution waits for the next due node,
     # recording it as the waiter, or ends because another execution already
-    # waits until then. See {Orchestrator::Policy.arm}.
+    # waits until then. A waiter whose execution waiter_alive reports ended
+    # is taken over before its record expires. See {Orchestrator::Policy.arm}.
     #
     # @param token [String] identifies the execution
     # @param cycle [Integer] times the execution has run due nodes already
@@ -215,7 +218,9 @@ module Glazed
       now = @clock.call
       wake = nil #: Time?
       @backend.update_wakeup do |record|
-        kept, wake = Orchestrator::Policy.arm(record, token:, next_due:, now:, gap: poll_seconds, grace: grace || longest_lease, cycle:, max_cycles:)
+        kept, wake = Orchestrator::Policy.arm(
+          record, token:, next_due:, now:, gap: poll_seconds, grace: grace || longest_lease, cycle:, max_cycles:, alive: @waiter_alive,
+        )
         kept
       end
       if wake

data/lib/glazed/dialects/{wlc.rb → aireos.rb}

@@ -13,15 +13,31 @@ module Glazed
   module Dialects
     # Cisco Wireless LAN Controller on AireOS. Ported from oxidized's aireos model.
     #
-    # AireOS asks for the user name and password inside the session, after SSH
-    # authentication as well.
-    class Wlc < Base
+    # Depending on version and settings, AireOS asks for the user name and
+    # password again inside the session after SSH authentication.
+    class Aireos < Base
+      # Encrypted secret in the running configuration: key index, IV, HMAC, optional length and ciphertext.
+      ENCRYPTED = /1 \h{32} \h{40}(?: \d+)? \h+/ #: Regexp
+
       prompt(/^\([^)]+\)\s>/)
       comment_prefix '# '
 
-      sensitive_pattern(/^config snmp community create (\S+)/) { it[1] }
-      sensitive_pattern(/^config wlan security wpa akm psk set-key (?:ascii|hex) (\S+)/) { it[1] }
-      sensitive_pattern(/^config (?:radius|tacacs) (?:auth|acct|athr) add \d+ \S+ \d+ (?:ascii|hex) (\S+)/) { it[1] }
+      # Parts of `show run-config commands` page even after `config paging disable`.
+      interact(/--More-- or \(q\)uit/) do |cli, _buffer, match|
+        cli.write(' ')
+        Actions::Replace.new(content: match.pre_match + match.post_match)
+      end
+
+      sensitive_pattern(/(?<= )#{ENCRYPTED}/o) { it[0] }
+      # Some releases print run-config commands without the leading `config`.
+      sensitive_pattern(/^(?:config )?wlan security wpa akm \S+ set-key (?:ascii|hex) (?!encrypt )(\S+)/) { it[1] }
+      sensitive_pattern(/^(?:config )?(?:radius|tacacs) (?:auth|acct|athr) add \d+ \S+ \d+ (?:ascii|hex) (\S+)/) { it[1] }
+      sensitive_pattern(/^(?:config )?mgmtuser (?:add|password) (?!encrypt)\S+ (\S+)/) { it[1] }
+      sensitive_pattern(/^(?:config )?netuser (?:add|password) (?!encrypt)\S+ (\S+)/) { it[1] }
+      # Covers the password, its crypt hash and the enable secret.
+      sensitive_pattern(/^(?:config )?ap mgmtuser add (?:encrypt )?username \S+ password (.+) \S+$/) { it[1] }
+      sensitive_pattern(/^(?:config )?snmp community (?:create|mode \S+|accessmode \S+|ipaddr \S+ \S+|ipv6addr \S+ \S+) (\S+)/) { it[1] }
+      sensitive_pattern(/^(?:config )?snmp v3user create \S+ (?:ro|rw) \S+ \S+ (\S.*)$/) { it[1] }
 
       runbook :get do |book|
         book.command('show udi') { comment("#{clean(it.output)}\n") }

data/lib/glazed/dialects/aruba_instant.rb

@@ -0,0 +1,90 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+# Derived from Oxidized (https://github.com/ytti/oxidized) lib/oxidized/model/arubainstant.rb,
+# Copyright 2013-2015 Saku Ytti, Samer Abdel-Hafez and Oxidized contributors,
+# licensed under the Apache License, Version 2.0. Modified for the glazed dialect API.
+
+require 'glazed/dialects/base'
+
+module Glazed
+  module Dialects
+    # Aruba Instant (IAP and Instant virtual controller). Ported from oxidized's arubainstant model.
+    #
+    # Node secrets: "enable" is sent as the enable password; without it, param
+    # "enable" set to "true" runs `enable` without one.
+    class ArubaInstant < Base
+      # Access levels of `mgmt-user` on Instant virtual controllers and Mobility Access Switches.
+      MGMT_ROLES = 'root|guest-provisioning|network-operations|read-only|location-api-mgmt' #: String
+
+      # The prompt gains a leading space while a core file is present.
+      prompt(/^ ?[\w:.@-]+[#>] $/)
+      comment_prefix '# '
+
+      sensitive_pattern(/ipsec (\S+)$/) { it[1] }
+      sensitive_pattern(/community (\S+)$/) { it[1] }
+      sensitive_pattern(/^\s*snmp-server host [\d.]+ version 2c (\S+)/) { it[1] }
+      sensitive_pattern(/^\s*snmpv3-user .*?auth-pass (\S+)/) { it[1] }
+      sensitive_pattern(/^\s*snmpv3-user .*?priv-pass (\S+)/) { it[1] }
+      sensitive_pattern(/^\s*mgmt-user \S+ (?:#{MGMT_ROLES}) (\S+)$/) { it[1] }
+      sensitive_pattern(/^\s*mgmt-user \S+ (?!(?:#{MGMT_ROLES}) )(\S+)(?: (?:read-only|guest-mgmt))?$/) { it[1] }
+      sensitive_pattern(/key (\S+)$/) { it[1] }
+      sensitive_pattern(/^\s*(?:\S+-)?(?:passwd|passphrase|psk|secret|password|passwords) (\S+)$/) { it[1] }
+      sensitive_pattern(/^\s*user \S+ (\S+) \S+$/) { it[1] }
+      sensitive_pattern(/^\s*hash-mgmt-user \S+ password \S+ (\S+)(?: usertype .*)?$/) { it[1] }
+      sensitive_pattern(/^\s*hash-mgmt-user \S+ password (\S+)(?: usertype .*)?$/) { it[1] }
+
+      runbook :get do |book|
+        book.command('show version') do |result|
+          comment(result.output.each_line.grep_v(/^(?:Switch|AP) uptime is |^Reboot Time and Cause/).join)
+        end
+        book.command('show activate status') do |result|
+          comment("#{result.output.each_line.grep_v(/^(?:Activate |Provision interval|Cloud Activation Key)/).join}\n")
+        end
+        book.command('show aps') do |result|
+          comment("#{result.output.each_line.map { aps_line(it) }.join}\n")
+        end
+        book.command('show running-config no-encrypt')
+      end
+
+      communication :telnet do |config|
+        config.username_prompt = /^User:\s*/
+        config.password_prompt = /^Password:\s*/
+      end
+
+      communication :ssh, :telnet do |config|
+        config.post_login do |run|
+          if (enable = run.secret('enable'))
+            run.cli.write("enable\n")
+            run.cli.expect(/^[pP]assword:/)
+            run.cli.write("#{enable.value}\n")
+            run.cli.expect(run.cli.prompt)
+          elsif run.param('enable') == 'true'
+            run.cli.command('enable')
+          end
+        end
+        config.pre_logout do |run|
+          run.cli.write("exit\n") if run.secret('enable') || run.param('enable') == 'true'
+          run.cli.write("exit\n")
+        end
+      end
+
+      class << self
+        # Drops the per-radio columns of `show aps`, which change on their own.
+        #
+        # @param line [String] a line of `show aps`
+        # @return [String] the line with the volatile columns removed
+        #: (String line) -> String
+        def aps_line(line)
+          if line.match?(/^Name/)
+            line.sub(/^(Name +IP Address +).*(Type +IPv6 Address +).*(Serial #).*$/, '\1\2\3')
+          else
+            line.sub(/^(\S+ +\S+ +)(?:\S+ +){3}(\S+ +\S+ +)(?:\S+ +){2}(\S+) +.*$/, '\1\2\3')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/glazed/dialects/eos.rb

@@ -0,0 +1,66 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+# Derived from Oxidized (https://github.com/ytti/oxidized) lib/oxidized/model/eos.rb,
+# Copyright 2013-2015 Saku Ytti, Samer Abdel-Hafez and Oxidized contributors,
+# licensed under the Apache License, Version 2.0. Modified for the glazed dialect API.
+
+require 'glazed/dialects/base'
+
+module Glazed
+  module Dialects
+    # Arista EOS. Ported from oxidized's eos model.
+    #
+    # Node secrets: "enable" is sent as the enable password; without it, param
+    # "enable" set to "true" runs `enable` without one.
+    class Eos < Base
+      # Optional encoding prefix of a secret (0 plaintext, 5 and sha512 hashes, 7 and 8a reversible).
+      SECRET_TYPE = '(?:(?:0|5|7|8a|sha512) )?'
+
+      prompt(/^.+[#>]$/)
+      comment_prefix '! '
+
+      sensitive_pattern(/^snmp-server community (?:(?:0|7|8a) (?=\S+ (?:ro|rw)\b))?(\S+)/) { it[1] }
+      # SNMPv3 hosts carry a user name, not a community.
+      sensitive_pattern(/^snmp-server host (?!.* version 3 )\S+(?> vrf \S+)?(?> informs?| traps)?(?> version (?:1|2c))? +(\S+)/) { it[1] }
+      sensitive_pattern(/^snmp-server user .*? auth (?:md5|sha\d*) (?:(?:0|7|8a) )?(\S+)/) { it[1] }
+      sensitive_pattern(/^snmp-server user .*? priv \w+ (?:(?:0|7|8a) )?(\S+)/) { it[1] }
+      sensitive_pattern(/^enable (?:secret|password)(?: level \d+)? #{SECRET_TYPE}(\S+)/) { it[1] }
+      sensitive_pattern(/^username \S+ .*?(?:secret|password) #{SECRET_TYPE}(\S+)/) { it[1] }
+      sensitive_pattern(/^\s+(?:secret|password) #{SECRET_TYPE}(\S+)/) { it[1] }
+      sensitive_pattern(/^(?:tacacs|radius)-server (?:.+ )?key (?:(?:0|7|8a) )?(\S+)/) { it[1] }
+      sensitive_pattern(/^\s+server \S+(?: .+)? key (?:(?:0|7|8a) )?(\S+)/) { it[1] }
+      sensitive_pattern(/^\s*key (?:\h+ )?(?:0|7|8a) (\S+)/) { it[1] }
+      sensitive_pattern(/^\s*(?:encryption-key|key-string|pre-shared-key) (?:(?:0|7|8a) )?(\S+)/) { it[1] }
+      sensitive_pattern(/^\s*(?:isis |ip ospf )?authentication(?:-key| key) (?:(?:0|7|8a) )?(\S+)/) { it[1] }
+      sensitive_pattern(/^\s*ip ospf message-digest-key \d+ \S+ (?:(?:0|7|8a) )?(\S+)/) { it[1] }
+      sensitive_pattern(/^\s*neighbor \S+ password (?:(?:0|7|8a) )?(\S+)/) { it[1] }
+      sensitive_pattern(/^\s*vrrp \d+ authentication text (\S+)/) { it[1] }
+      sensitive_pattern(/^ntp authentication-key \d+ \S+ (?:(?:0|7|8a) )?(\S+)/) { it[1] }
+      sensitive_pattern(/^service unsupported-transceiver \S+ (\S+)/) { it[1] }
+      sensitive_pattern(/-(?:cv|ingest)auth=key,(\S+)/) { it[1] }
+
+      runbook :get do |book|
+        book.command('show inventory | no-more') { comment(it.output) }
+        book.command('show running-config | no-more | exclude ! Time:')
+      end
+
+      communication :ssh, :telnet do |config|
+        config.post_login do |run|
+          if (enable = run.secret('enable'))
+            run.cli.write("enable\n")
+            run.cli.expect(/[pP]assword:\s?$/)
+            run.cli.write("#{enable.value}\n")
+            run.cli.expect(run.cli.prompt)
+          elsif run.param('enable') == 'true'
+            run.cli.command('enable')
+          end
+          run.cli.command('terminal length 0')
+        end
+        config.pre_logout { |run| run.cli.write("exit\n") }
+      end
+    end
+  end
+end

data/lib/glazed/dialects/nec_ix.rb

@@ -26,7 +26,8 @@ module Glazed
       sensitive_pattern(/^ikev2 authentication psk id .+ key \S+ (\S+).*/) { it[1] }
       sensitive_pattern(/^ike policy ike-policy .+ key \S+ (\S+).*/) { it[1] }
       sensitive_pattern(/^radius host .+ key \S+ (\S+).*/) { it[1] }
-      sensitive_pattern(/^\s*authentication (?:secret-)?password \S+ (\S+)/) { it[1] }
+      sensitive_pattern(/^\s*authentication password \S+ (\S+)/) { it[1] }
+      sensitive_pattern(/^\s*authentication secret-password \d+ (.+?)\s*$/) { it[1] }
       sensitive_pattern(/^\s*ngn radius-auth password \S+ (\S+)/) { it[1] }
       sensitive_pattern(/^\s*usbmem authentication .+ password-file \S+ \S+ (\S+)/) { it[1] }
       sensitive_pattern(/^web-auth username \S+ (?:secret-)?password (\S+)/) { it[1] }

data/lib/glazed/dialects/routeros.rb

@@ -0,0 +1,92 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+# Derived from Oxidized (https://github.com/ytti/oxidized) lib/oxidized/model/routeros.rb,
+# Copyright 2013-2015 Saku Ytti, Samer Abdel-Hafez and Oxidized contributors,
+# licensed under the Apache License, Version 2.0. Modified for the glazed dialect API.
+
+require 'glazed/dialects/base'
+
+module Glazed
+  module Dialects
+    # MikroTik RouterOS. Ported from oxidized's routeros model.
+    #
+    # Node params: "routeros_hide_sensitive" set to "true" collects `/export hide-sensitive`.
+    # Otherwise RouterOS 7 and later run `/export show-sensitive`, and older versions `/export`.
+    class Routeros < Base
+      # A key=value value, either quoted or bare.
+      VALUE = /(?:"((?:[^"\\]|\\.)*)"|([^\s"]\S*))/ #: Regexp
+
+      prompt(/\[\w+@\S+(\s+\S+)*\]\s?>\s?$/)
+      comment_prefix '# '
+
+      # Cursor movements also appear, such as before the prompt.
+      filter_output { it.gsub(/\x1B\[[0-9;?]*[A-Za-z]/, '') }
+      filter_output { it.gsub(/^\r+(.+)/, '\1').gsub(/([^\r]*)\r+$/, '\1') }
+
+      sensitive_pattern(
+        /(?<![\w.-])[\w.-]*(?:password|passphrase|secret|pre-shared-key|preshared-key|private-key|psk|md5-key|auth-key|authentication-key|protection-key|static-key-\d|trap-community)=#{VALUE}/,
+      ) { it[1] || it[2] }
+      # Community names follow `name=`, one section at a time. \G continues from the previous match.
+      sensitive_pattern(
+        /(?:^\/snmp community\n|\G(?<=\S)[^\n]*\n)(?:(?:add|set)[^\n]*\n)*?(?:add|set) [^\n]*?(?<=\s)name=#{VALUE}/,
+      ) { it[1] || it[2] }
+
+      runbook :get do |book|
+        book.command('/system resource print') do |result|
+          comment(result.output.each_line.grep(/(version|factory-software|total-memory|cpu|cpu-count|total-hdd-space|architecture-name|board-name|platform):/).join)
+        end
+        book.raw('/system package update print') do |run|
+          line = run.command('/system package update print').output.each_line.grep(/installed-version:\s|current-version:\s/).first
+          next unless line
+
+          run.vars[:ros_version] = line[/:\s*(\d+)/, 1]&.to_i
+          run.emit(comment(line))
+        end
+        book.command('/system history print without-paging') { comment(it.output) }
+        book.raw('/export') do |run|
+          line = if run.param('routeros_hide_sensitive') == 'true'
+            '/export hide-sensitive'
+          elsif (version = run.vars[:ros_version]).is_a?(Integer) && version >= 7
+            '/export show-sensitive'
+          else
+            '/export'
+          end
+          run.emit(export(run.command(line).output))
+        end
+      end
+
+      communication :telnet do |config|
+        config.username_prompt = /^Login:/
+        config.password_prompt = /^Password:/
+        config.pre_logout { |run| run.cli.write("quit\n") }
+      end
+
+      communication :ssh do |config|
+        config.exec = true
+      end
+
+      class << self
+        # Drops the lines of `/export` that change without configuration changes
+        # and joins the lines RouterOS wraps with a trailing backslash.
+        #
+        # @param output [String] output of `/export`
+        # @return [String] the configuration, ending with a newline
+        #: (String output) -> String
+        def export(output)
+          config = output
+                   .gsub(/\\\r?\n\s+/, '')
+                   .gsub("# inactive time\n", '')
+                   .gsub(/# received packet from \S+ bad format\n/, '')
+                   .gsub("# poe-out status: short_circuit\n", '')
+                   .gsub("# Firmware upgraded successfully, please reboot for changes to take effect!\n", '')
+                   .gsub(/# \S+ not ready\n/, '')
+          lines = config.split("\n").reject { it.match?(%r{^#\s\w{3}/\d{2}/\d{4}.*$}) || it.match?(/^#\s\d{4}-\d{2}-\d{2}.*$/) }
+          "#{lines.join("\n")}\n"
+        end
+      end
+    end
+  end
+end

data/lib/glazed/node_session.rb

@@ -0,0 +1,152 @@
+# typed: strict
+# frozen_string_literal: true
+
+# SPDX-License-Identifier: Apache-2.0
+
+require 'glazed'
+require 'glazed/cli_session'
+require 'glazed/dialect/run'
+require 'glazed/dialects'
+require 'glazed/proto'
+require 'glazed/secrets_resolver'
+require 'glazed/transports'
+
+module Glazed
+  # Logged-in session to one node: connects, answers the dialect's login
+  # prompts, runs its post-login hooks, and later its pre-logout hooks.
+  #
+  # @example
+  #   session = Glazed::NodeSession.new(node:)
+  #   session.open
+  #   begin
+  #     session.run.command('show version')
+  #     session.logout
+  #   ensure
+  #     session.close
+  #   end
+  class NodeSession
+    # Transport used when a node names none.
+    DEFAULT_TRANSPORT = 'ssh' #: String
+
+    # @param node [Proto::V1::Node] node to connect to
+    # @param transport [Transports::Base, nil] transport to use instead of building one from the node
+    # @param timeout [Numeric] seconds to wait for each expected output
+    # @param exec [Boolean, nil] run commands on exec channels when the transport supports it; the dialect decides when nil
+    # @param interactions [Array<Dialect::Interaction>] responders tried after the dialect's own
+    # @param progress [Proc] receives a message when connecting and whenever waiting for the device
+    # @param secrets [SecretsResolver] reads the node's password and secrets
+    #: (node: Proto::V1::Node, ?transport: Transports::Base?, ?timeout: Numeric, ?exec: bool?, ?interactions: Array[Dialect::Interaction], ?progress: ^(String) -> void, ?secrets: SecretsResolver) -> void
+    def initialize(node:, transport: nil, timeout: CliSession::DEFAULT_TIMEOUT, exec: nil, interactions: [], progress: ->(_message) {}, secrets: SecretsResolver.new)
+      @node = node #: Proto::V1::Node
+      @secrets = secrets #: SecretsResolver
+      @progress = progress #: ^(String) -> void
+      @dialect = Dialects.find(node.dialect) #: singleton(Dialects::Base)
+      @kind = (node.transport.empty? ? DEFAULT_TRANSPORT : node.transport) #: String
+      @transport = transport || build_transport #: Transports::Base
+      @config = @dialect.communication_config(@kind.to_sym) #: Dialect::CommunicationConfig
+      @exec = (exec.nil? ? @config.exec : exec) #: bool
+      @cli = CliSession.new(
+        transport: @transport,
+        prompt: @dialect.prompt_pattern,
+        data_filters: @dialect.data_filters,
+        interactions: @dialect.interactions + interactions,
+        exec: @exec,
+        timeout:,
+        progress:,
+      ) #: CliSession
+      @run = Dialect::Run.new(cli: @cli, node:, dialect: @dialect, secrets:) #: Dialect::Run
+    end
+
+    #: Proto::V1::Node
+    attr_reader :node
+    #: singleton(Dialects::Base)
+    attr_reader :dialect
+    # @return [Transports::Base] underlying transport
+    #: Transports::Base
+    attr_reader :transport
+    # @return [CliSession] session to the node
+    #: CliSession
+    attr_reader :cli
+    # @return [Dialect::Run] run state passed to runbooks and hooks
+    #: Dialect::Run
+    attr_reader :run
+
+    # Connects, logs in and runs the post-login hooks. The connection is
+    # closed when any of them fails.
+    #
+    # @return [void]
+    # @raise [TimeoutError] when the device does not answer as the dialect expects
+    # @raise [ConnectionClosed] when the device drops the connection
+    #: () -> void
+    def open
+      @transport.pty_options = @config.pty_options
+      @progress.call("connecting to #{@transport.host} over #{@kind}")
+      @transport.open
+      begin
+        login unless exec_only?
+        @config.post_login_hooks.each { it.call(run) }
+      rescue StandardError
+        close
+        raise
+      end
+    end
+
+    # Runs the pre-logout hooks.
+    #
+    # @return [void]
+    #: () -> void
+    def logout
+      @config.pre_logout_hooks.each { it.call(run) }
+    end
+
+    # Closes the connection without logging out. Safe to call more than once.
+    #
+    # @return [void]
+    #: () -> void
+    def close
+      @transport.close
+    end
+
+    # @return [Boolean] whether every command runs on exec channels, so the shell is never used
+    #: () -> bool
+    private def exec_only? = @exec && @transport.exec?
+
+    # Answers login prompts the dialect declares until the first prompt appears.
+    # Each login prompt is answered at most once, and none has to appear, as
+    # some devices ask for credentials only under certain settings.
+    #
+    # @return [void]
+    #: () -> void
+    private def login
+      answers = {} #: Hash[Regexp, ^() -> String?]
+      if (username_prompt = @config.username_prompt)
+        answers[username_prompt] = -> { node.username }
+      end
+      if (password_prompt = @config.password_prompt)
+        answers[password_prompt] = -> { @secrets.maybe(node.password)&.value }
+      end
+      # Sorbet cannot check a splat against expect's keyword options (7019).
+      cli = @cli #: as untyped
+      loop do
+        answer = answers.delete(cli.expect(*answers.keys, @cli.prompt).regexp)
+        return unless answer
+
+        @cli.write("#{answer.call}\n")
+      end
+    end
+
+    # @return [Transports::Base] transport built from the node
+    # @raise [Finder::NotFound] when the transport is unknown
+    #: () -> Transports::Base
+    private def build_transport
+      Transports.find(@kind).new(
+        host: node.host.empty? ? node.name : node.host,
+        port: node.port.zero? ? nil : node.port,
+        username: node.username,
+        password: @secrets.maybe(node.password),
+        transport_options: node.transport_options,
+        secrets: @secrets,
+      )
+    end
+  end
+end

data/lib/glazed/orchestrator/policy.rb

@@ -149,7 +149,8 @@ module Glazed
       # Decides whether a coordinator execution waits for the next due node.
       # It does unless another live execution already waits until then, or
       # it has looped max_cycles times; a waiter that stops waiting gives up
-      # its record.
+      # its record. A waiter that ended without giving it up, such as by
+      # failing, is live until its record expires unless alive tells otherwise.
       #
       # @param record [Proto::V1::Wakeup, nil] the current waiter
       # @param token [String] identifies the deciding execution
@@ -159,17 +160,20 @@ module Glazed
       # @param grace [Integer] seconds after waking until another execution may take over
       # @param cycle [Integer] times the execution has run due nodes already
       # @param max_cycles [Integer] cycles after which the execution ends
+      # @param alive [Proc] tells whether the execution of a waiter's token still runs
       # @return [Array(Proto::V1::Wakeup, Time), Array(Proto::V1::Wakeup, nil), Array(nil, nil)] the waiter
       #   record to keep, and when to wake; nil when the execution ends
-      #: (Proto::V1::Wakeup? record, token: String, next_due: Time?, now: Time, gap: Integer, grace: Integer, cycle: Integer, max_cycles: Integer) -> [Proto::V1::Wakeup?, Time?]
-      def self.arm(record, token:, next_due:, now:, gap:, grace:, cycle:, max_cycles:)
+      #: (Proto::V1::Wakeup? record, token: String, next_due: Time?, now: Time, gap: Integer, grace: Integer, cycle: Integer, max_cycles: Integer, ?alive: ^(String) -> bool) -> [Proto::V1::Wakeup?, Time?]
+      def self.arm(record, token:, next_due:, now:, gap:, grace:, cycle:, max_cycles:, alive: ->(_token) { true })
         own = record&.owner == token
         return [own ? nil : record, nil] if next_due.nil? || cycle >= max_cycles
 
         wake = [next_due, cycle.zero? ? now : now + gap].max
         expires_at = time(record&.expires_at)
         wake_at = time(record&.wake_at)
-        return [record, nil] if !own && expires_at && expires_at > now && wake_at && wake_at <= wake
+        if record && !own && expires_at && expires_at > now && wake_at && wake_at <= wake && alive.call(record.owner)
+          return [record, nil]
+        end
 
         [Proto::V1::Wakeup.new(owner: token, wake_at: timestamp(wake), expires_at: timestamp(wake + grace)), wake]
       end

data/lib/glazed/steps/publish.rb

@@ -10,15 +10,12 @@ require 'glazed/storages'
 
 module Glazed
   module Steps
-    # Batch: writes an artifact of every node into a storage as one change,
-    # such as one commit naming the nodes.
+    # Batch: writes an artifact of every node into a storage, as one change
+    # per changed node, such as a commit titled "glazed: rt1".
     #
     # @example Step definition
     #   {"name": "publish", "publish": {"input": "redacted", "storage": "team"}}
     class Publish < BatchStep
-      # Node names listed in a commit message before it is shortened.
-      MESSAGE_NAMES = 5 #: Integer
-
       # @param name [String] step name
       # @param input [String] artifact to read from each node
       # @param storage_name [String] storage to write to, by name
@@ -71,7 +68,7 @@ module Glazed
           names[key] = node.name
           [key, @context.read(artifact(state, @input))] #: Storages::Base::file
         end
-        results = @storage.write_all(files, message: ->(changed_keys) { message(changed_keys.map { names.fetch(it, it) }) })
+        results = @storage.write_all(files, message: ->(key) { "glazed: #{names.fetch(key, key)}" })
         output = @output
         return batch unless output
 
@@ -80,15 +77,6 @@ module Glazed
         end
         Proto::V1::BatchState.new(nodes:)
       end
-
-      # @param names [Array<String>] names of the nodes whose content changed
-      # @return [String] commit message naming the nodes
-      #: (Array[String] names) -> String
-      private def message(names)
-        listed = names.first(MESSAGE_NAMES).join(', ')
-        rest = names.size - MESSAGE_NAMES
-        rest.positive? ? "#{name}: update #{listed} and #{rest} more" : "#{name}: update #{listed}"
-      end
     end
   end
 end

data/lib/glazed/storages/base.rb

@@ -21,7 +21,7 @@ module Glazed
       class Conflict < Glazed::Error; end
 
       #: type file = [String, String]
-      #: type message_builder = ^(Array[String] changed_keys) -> String
+      #: type message_builder = ^(String changed_key) -> String
 
       # Outcome of writing one file.
       class WriteResult
@@ -78,15 +78,16 @@ module Glazed
       # @raise [InvalidKey] for an unsafe key
       #: (key: String, content: String, message: String) -> WriteResult
       def write(key:, content:, message:)
-        write_all([[key, content]], message: ->(_changed_keys) { message }).fetch(0)
+        write_all([[key, content]], message: ->(_changed_key) { message }).fetch(0)
       end
 
-      # Writes several files as one change where the storage supports it. Files
-      # whose content equals what their key holds are not written again.
+      # Writes several files, as one change per changed file where the storage
+      # records changes, such as one commit each. Files whose content equals
+      # what their key holds are not written again.
       #
       # @param files [Array<Array(String, String)>] keys and contents
-      # @param message [Proc] builds the description of the change, such as a
-      #   commit message, from the keys whose content changed; not called when none did
+      # @param message [Proc] builds the description of the change to one key,
+      #   such as a commit message; called only for keys whose content changed
       # @return [Array<WriteResult>] results, in the order of files
       # @raise [InvalidKey] for an unsafe key
       #: (Array[Base::file] files, message: Base::message_builder) -> Array[WriteResult]

data/lib/glazed/storages/git.rb

@@ -12,8 +12,8 @@ require 'glazed/storages/base'
 
 module Glazed
   module Storages
-    # Local git repository driven by the git command. Each write is one commit
-    # on the configured branch, optionally pulled before and pushed after.
+    # Local git repository driven by the git command. Each changed file is one
+    # commit on the configured branch, optionally pulled before and pushed after.
     #
     # @example
     #   storage = Glazed::Storages::Git.new(path: '/var/lib/glazed/raw')
@@ -81,12 +81,13 @@ module Glazed
       #: String?
       attr_reader :subdirectory
 
-      # Writes files in one commit. Nothing is committed when no content
-      # changed, and the pointers then name the existing HEAD.
+      # Writes each changed file in its own commit and pushes once. Nothing is
+      # committed when no content changed, and the pointers then name the
+      # existing HEAD.
       #
       # @param files [Array<Array(String, String)>] keys and contents
-      # @param message [Proc] builds the commit message from the changed keys
-      # @return [Array<WriteResult>] the commit holding each file, and whether the file changed
+      # @param message [Proc] builds the commit message for a changed key
+      # @return [Array<WriteResult>] the last commit, which holds every file, and whether each file changed
       # @raise [InvalidKey] for an unsafe key
       # @raise [CommandFailed] when a git command fails
       #: (Array[Base::file] files, message: Base::message_builder) -> Array[WriteResult]
@@ -104,7 +105,7 @@ module Glazed
           changed_paths = git(['diff', '--cached', '--name-only', '-z', '--', *paths]).split("\0")
           changed = keys.select { changed_paths.include?(repository_path(it)) }
           unless changed.empty?
-            git(['commit', '--quiet', '--no-verify', '-m', message.call(changed)])
+            changed.each { git(['commit', '--quiet', '--no-verify', '-m', message.call(it), '--', repository_path(it)]) }
             git(['push', '--quiet', Must.expect(remote), "HEAD:refs/heads/#{branch}"]) if remote
           end
           commit = git(%w[rev-parse HEAD]).strip

data/lib/glazed/storages/github.rb

@@ -15,8 +15,8 @@ require 'glazed/version'
 
 module Glazed
   module Storages
-    # GitHub repository written through the Git Data API. Each write is one
-    # commit on the branch, which is created from the base branch when missing.
+    # GitHub repository written through the Git Data API. Each changed file is
+    # one commit on the branch, which is created from the base branch when missing.
     #
     # @example
     #   storage = Glazed::Storages::Github.new(repository: 'org/netconf', branch: 'glazed/auto', token:)
@@ -99,12 +99,13 @@ module Glazed
       #: String?
       attr_reader :subdirectory
 
-      # Writes the changed files in one commit. Nothing is committed when no
-      # content changed, and the pointers then name the existing head.
+      # Writes each changed file in its own commit and moves the branch once.
+      # Nothing is committed when no content changed, and the pointers then
+      # name the existing head.
       #
       # @param files [Array<Array(String, String)>] keys and contents
-      # @param message [Proc] builds the commit message from the changed keys
-      # @return [Array<WriteResult>] the commit holding each file, and whether the file changed
+      # @param message [Proc] builds the commit message for a changed key
+      # @return [Array<WriteResult>] the last commit, which holds every file, and whether each file changed
       # @raise [InvalidKey] for an unsafe key
       # @raise [ApiError] when the API fails, or the branch keeps moving under concurrent writers
       #: (Array[Base::file] files, message: Base::message_builder) -> Array[WriteResult]
@@ -147,8 +148,8 @@ module Glazed
       end
 
       # @param files [Array<Array(String, String)>] keys and contents
-      # @param message [Proc] builds the commit message from the changed keys
-      # @return [Array(String, Array<String>)] SHA of the commit holding the files, and the changed keys
+      # @param message [Proc] builds the commit message for a changed key
+      # @return [Array(String, Array<String>)] SHA of the last commit, and the changed keys
       # @raise [ApiError] when the API fails
       #: (Array[Base::file] files, Base::message_builder message) -> [String, Array[String]]
       private def commit_files(files, message)
@@ -158,16 +159,18 @@ module Glazed
         existing = blob_shas(base_tree)
         candidates = existing ? files.reject { |key, content| existing[repository_path(key)] == blob_sha(content) } : files
 
-        commit = parent
+        commit = parent #: String
+        tree = base_tree
         changed = [] #: Array[String]
-        unless candidates.empty?
-          entries = candidates.map { |key, content| {path: repository_path(key), mode: '100644', type: 'blob', content:} }
-          tree = json(post("/repos/#{repository}/git/trees", {base_tree:, tree: entries})).fetch('sha')
-          unless tree == base_tree
-            changed = candidates.map(&:first)
-            body = {message: message.call(changed), tree:, parents: [parent], author: @author}.compact
-            commit = json(post("/repos/#{repository}/git/commits", body)).fetch('sha')
-          end
+        candidates.each do |key, content|
+          entry = {path: repository_path(key), mode: '100644', type: 'blob', content:}
+          new_tree = json(post("/repos/#{repository}/git/trees", {base_tree: tree, tree: [entry]})).fetch('sha')
+          next if new_tree == tree
+
+          body = {message: message.call(key), tree: new_tree, parents: [commit], author: @author}.compact
+          commit = json(post("/repos/#{repository}/git/commits", body)).fetch('sha')
+          tree = new_tree
+          changed << key
         end
 
         if head.nil?

data/lib/glazed/transports/base.rb

@@ -101,7 +101,7 @@ module Glazed
       # Runs one command outside the interactive session.
       #
       # @param command [String] command line
-      # @param timeout [Numeric] seconds to wait for the command to finish
+      # @param timeout [Numeric] seconds the command may stay silent before it finishes
       # @return [String] command output
       # @raise [NotImplementedError] when the transport has no exec
       #: (String command, timeout: Numeric) -> String

data/lib/glazed/transports/ssh.rb

@@ -58,6 +58,9 @@ module Glazed
           password: password&.value,
           timeout: connect_timeout,
           non_interactive: true,
+          # Some devices leave keepalives unanswered while busy, and the idle
+          # timeout of CliSession and #exec already detects a dead connection.
+          keepalive_maxcount: 0,
         )
       end
 
@@ -140,25 +143,29 @@ module Glazed
       # Runs a command on its own exec channel.
       #
       # @param command [String] command line
-      # @param timeout [Numeric] seconds to wait for the channel to close
+      # @param timeout [Numeric] seconds the channel may stay silent before it closes
       # @return [String] combined stdout and stderr
       # @raise [Glazed::Error] when the device refuses the exec request
-      # @raise [Glazed::TimeoutError] when the command does not finish in time
+      # @raise [Glazed::TimeoutError] when the channel stays silent for timeout seconds
       #: (String command, timeout: Numeric) -> String
       def exec(command, timeout:)
         output = +''
         closed = false #: bool
+        deadline = now + timeout.to_f
+        receive = lambda do |data|
+          output << data
+          deadline = now + timeout.to_f
+        end
         session.open_channel do |channel|
           channel.exec(command) do |_, success|
             raise Glazed::Error, "#{host}: exec request refused for #{command.inspect}" unless success
           end
-          channel.on_data { |_, data| output << data }
-          channel.on_extended_data { |_, _, data| output << data }
+          channel.on_data { |_, data| receive.call(data) }
+          channel.on_extended_data { |_, _, data| receive.call(data) }
           channel.on_close { closed = true }
         end
-        deadline = now + timeout.to_f
         session.loop(0.1) do
-          raise Glazed::TimeoutError, "#{host}: #{command.inspect} did not finish in #{timeout}s" if !closed && now > deadline
+          raise Glazed::TimeoutError, "#{host}: #{command.inspect} sent no output for #{timeout}s" if !closed && now > deadline
 
           !closed
         end

data/lib/glazed/version.rb

@@ -5,5 +5,5 @@
 
 module Glazed
   # Gem version.
-  VERSION = '0.2.0' #: String
+  VERSION = '0.3.0' #: String
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: glazed
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Sorah Fukumori
@@ -164,14 +164,18 @@ files:
 - lib/glazed/dialect/runbook.rb
 - lib/glazed/dialect/sensitive_pattern.rb
 - lib/glazed/dialects.rb
+- lib/glazed/dialects/aireos.rb
+- lib/glazed/dialects/aruba_instant.rb
 - lib/glazed/dialects/awplus.rb
 - lib/glazed/dialects/base.rb
+- lib/glazed/dialects/eos.rb
 - lib/glazed/dialects/ios.rb
 - lib/glazed/dialects/junos.rb
 - lib/glazed/dialects/nec_ix.rb
-- lib/glazed/dialects/wlc.rb
+- lib/glazed/dialects/routeros.rb
 - lib/glazed/finder.rb
 - lib/glazed/must.rb
+- lib/glazed/node_session.rb
 - lib/glazed/orchestrator.rb
 - lib/glazed/orchestrator/policy.rb
 - lib/glazed/plugin_registry.rb