runsheets 0.0.1

data/lib/runsheets/config.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "myway_config"
+
+module Runsheets
+  class ConfigError < Error; end
+
+  # Settings for the `runsheets` command, layered lowest to highest:
+  #
+  #   1. lib/runsheets/config/defaults.yml bundled with the gem (the schema; see that file)
+  #   2. the XDG user config, ~/.config/runsheets/runsheets.yml (myway_config)
+  #   3. the project config, ./config/runsheets.yml, or the file named by
+  #      --config FILE or RUNSHEETS_CONFIG in its place
+  #   4. RUNSHEETS_<KEY> environment variables
+  #   5. the overrides hash, which the CLI fills from the command line
+  #
+  # Every key in defaults.yml is an attribute here (dir, port, bind, runs_dir,
+  # open, check, init, dump, engineer, why, log_level, quiet). Paths are
+  # expanded; flags accept 1/true/yes/on.
+  class Config < MywayConfig::Base
+    config_name :runsheets
+    env_prefix  :runsheets
+    defaults_path File.expand_path("config/defaults.yml", __dir__)
+
+    TRUE_WORDS = %w[1 true yes on].freeze
+
+    class << self
+      def blank?(value) = value.nil? || value.to_s.strip.empty?
+
+      # A coercion that hands the stripped text to the block, or gives nil for
+      # a blank so the bundled default fills it (see fill_blanks_from_defaults).
+      def unless_blank = ->(v) { blank?(v) ? nil : yield(v.to_s.strip) }
+
+      def whole_number(text)
+        port = Integer(text, 10)
+        raise ConfigError, "port must be between 1 and 65535, got #{port}" unless (1..65_535).cover?(port)
+
+        port
+      rescue ArgumentError
+        raise ConfigError, "port must be a whole number, got #{text.inspect}"
+      end
+
+      # The XDG user config: the file myway_config found, else where it would be.
+      def xdg_path
+        MywayConfig::Loaders::XdgConfigLoader.find_config_file(config_name) ||
+          File.join(Dir.home, ".config", config_name.to_s, "#{config_name}.yml")
+      end
+
+      # The project config read when neither --config nor RUNSHEETS_CONFIG names a file.
+      def project_path = File.expand_path(Anyway::Settings.default_config_path.call(config_name))
+
+      # The bundled defaults, coerced, keyed by attribute. What the help text shows.
+      def bundled_defaults = schema.to_h { |key, value| [key, type_caster.coerce(key, value)] }
+
+      # Any RACK_ENV is fine; the environment sections of defaults.yml are optional.
+      def validate_environment! = nil
+    end
+
+    # Coercions: the raw YAML/env/CLI value becomes the type the attribute needs.
+    TEXT = unless_blank(&:itself)
+    PATH = unless_blank { File.expand_path(it) }
+    PORT = unless_blank { whole_number(it) }
+    FLAG = ->(v) { v == true || TRUE_WORDS.include?(v.to_s.strip.downcase) }
+
+    attr_config :dir, :port, :bind, :runs_dir, :open, :check, :init, :dump, :engineer, :why, :log_level, :quiet
+    coerce_types dir: PATH, runs_dir: PATH, bind: TEXT, port: PORT, open: FLAG, check: FLAG, init: FLAG, dump: FLAG,
+                 engineer: TEXT, why: TEXT, log_level: unless_blank { SessionLog.level_name(it) }, quiet: FLAG
+
+    on_load :fill_blanks_from_defaults
+
+    # The config file read in place of the project config, set while loading.
+    attr_reader :path
+
+    # overrides: a Hash of values that beat every other layer (the CLI's).
+    # path: the config file to read instead of RUNSHEETS_CONFIG / ./config/runsheets.yml.
+    def initialize(overrides = nil, path: nil)
+      @explicit_path = path
+      @override_keys = overrides.to_h.keys.map(&:to_s)
+      super(overrides&.transform_keys(&:to_s))
+    end
+
+    # The config files that exist and were read, lowest layer first.
+    def files = [self.class.xdg_path, path].select { File.file?(it) }
+
+    # The bind address in force when it came from the project config found
+    # in the current directory, else nil. That file arrives with whatever
+    # repository was cloned, so the CLI will not let it alone open the
+    # server to the network (see CLI.check_bind_source).
+    def project_bind
+      return nil if @explicit_path || !self.class.blank?(ENV["RUNSHEETS_CONFIG"])
+      return nil if @override_keys.include?("bind") || !self.class.blank?(ENV["RUNSHEETS_BIND"])
+      return nil unless File.file?(path.to_s)
+
+      data = YAML.safe_load_file(path, aliases: false)
+      data.is_a?(Hash) && data.key?("bind") ? bind : nil
+    rescue Psych::Exception
+      nil
+    end
+
+    # The settings in force as the flat hash a config file holds. dump is
+    # left out: it is an action for this run, and a saved `dump: true` would
+    # make every later run print and exit; check and init likewise. why is
+    # left out too: it belongs to one session, and a saved one would start
+    # every later session with it.
+    def settings = (self.class.config_attributes - %i[dump check init why]).to_h { [it.to_s, public_send(it)] }
+
+    # The settings in force as the text of a config file: a comment header,
+    # then one key per setting. `runsheets --dump` prints this.
+    def to_config_yaml
+      <<~YAML + settings.to_yaml.delete_prefix("---\n")
+        # runsheets settings, written by `runsheets --dump` on #{Time.now.strftime('%Y-%m-%d %H:%M')}.
+        # One key per setting; RUNSHEETS_* variables and command-line options
+        # override these. See `runsheets --help` for the layers.
+      YAML
+    end
+
+    private
+
+    # Where anyway_config's yml loader reads from. A file named explicitly
+    # (option or variable) must exist; the project config may be absent.
+    def resolve_config_path(_name, _env_prefix)
+      explicit = @explicit_path
+      explicit = ENV["RUNSHEETS_CONFIG"] if self.class.blank?(explicit)
+      explicit = nil if self.class.blank?(explicit)
+      raise ConfigError, "config file not found: #{explicit}" if explicit && !File.file?(File.expand_path(explicit))
+
+      @path = explicit ? File.expand_path(explicit) : self.class.project_path
+    end
+
+    # A blank in any layer (RUNSHEETS_PORT="" say) coerces to nil; put the
+    # bundled default back so an attribute is never accidentally unset.
+    def fill_blanks_from_defaults
+      self.class.bundled_defaults.each do |key, value|
+        public_send(:"#{key}=", value) if public_send(key).nil?
+      end
+    end
+  end
+end

data/lib/runsheets/diff.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Runsheets
+  # A small line diff, enough to show how a block's code changed between a
+  # recorded run and the runbook as it is now, or how a block's output
+  # differs from its expect block. Standard library only.
+  module Diff
+    Line = Data.define(:tag, :text) do
+      def unchanged? = tag == " "
+      def removed?   = tag == "-"
+      def added?     = tag == "+"
+      def to_s       = "#{tag}#{text}"
+    end
+
+    SAME = ->(a, b) { a == b }
+
+    # Lines of +before+ and +after+ tagged " " (same), "-" (only in before)
+    # or "+" (only in after), in order. Longest-common-subsequence based.
+    def self.lines(before, after)
+      ops(before.to_s.lines(chomp: true), after.to_s.lines(chomp: true)).map do |tag, old, new|
+        Line.new(tag, tag == "+" ? new : old)
+      end
+    end
+
+    # The edit script between two arrays of lines, one [tag, before_line,
+    # after_line] per row: " " when +same+ says the pair is the same, "-"
+    # with a nil after_line, "+" with a nil before_line.
+    def self.ops(before, after, same: SAME)
+      table = lcs_table(before, after, same)
+      walk(before, after, table, same)
+    end
+
+    # Unified-style text, one tagged line per row.
+    def self.unified(before, after) = lines(before, after).join("\n")
+
+    def self.changed?(before, after) = before.to_s != after.to_s
+
+    def self.lcs_table(before, after, same)
+      table = Array.new(before.size + 1) { Array.new(after.size + 1, 0) }
+      before.each_index.reverse_each do |i|
+        after.each_index.reverse_each do |j|
+          table[i][j] = same.call(before[i], after[j]) ? table[i + 1][j + 1] + 1 : [table[i + 1][j], table[i][j + 1]].max
+        end
+      end
+      table
+    end
+    private_class_method :lcs_table
+
+    def self.walk(before, after, table, same)
+      out = []
+      i = j = 0
+      while i < before.size && j < after.size
+        tag, di, dj = same.call(before[i], after[j]) ? [" ", 1, 1] : choose(table, i, j)
+        out << [tag, (before[i] unless tag == "+"), (after[j] unless tag == "-")]
+        i += di
+        j += dj
+      end
+      out.concat(before[i..].map { ["-", it, nil] }).concat(after[j..].map { ["+", nil, it] })
+    end
+    private_class_method :walk
+
+    # Which row to emit at (i, j) when the lines there differ: [tag,
+    # advance before, advance after].
+    def self.choose(table, i, j) = table[i + 1][j] >= table[i][j + 1] ? ["-", 1, 0] : ["+", 0, 1]
+    private_class_method :choose
+  end
+end

data/lib/runsheets/execution.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+module Runsheets
+  # One execution of a block: what ran, where its output went, how it ended.
+  # Executor drives the state transitions; everything else only reads.
+  class Execution
+    STATES = %i[pending running finished timed_out stopped failed].freeze
+
+    # tee is where the output also goes as it arrives: the session log's
+    # stream for this execution (anything with #write and #close), or nil.
+    attr_reader :id, :block_id, :step_slug, :command, :cmd_path, :log_path, :timeout, :tee,
+                :state, :pid, :started_at, :finished_at, :exit_status, :signal, :error
+
+    def initialize(id:, block_id:, step_slug:, command:, cmd_path:, log_path:, timeout: nil, background: false, tee: nil)
+      @id         = id
+      @block_id   = block_id
+      @step_slug  = step_slug
+      @command    = Array(command).map(&:to_s)
+      @cmd_path   = cmd_path
+      @log_path   = log_path
+      @timeout    = timeout
+      @background = background
+      @tee        = tee
+      @state      = :pending
+      @stop       = false
+      @mutex      = Mutex.new
+      @thread     = nil
+      @on_end     = []
+    end
+
+    def background? = @background
+    def pending?    = state == :pending
+    def running?    = state == :running
+    def finished?   = %i[finished timed_out stopped failed].include?(state)
+    def timed_out?  = state == :timed_out
+    def stopped?    = state == :stopped
+    def success?    = state == :finished && exit_status == 0
+    def failure?    = finished? && !success? && !stopped?
+
+    def stop_requested? = @stop
+
+    # Call +block+ with this execution once it has ended, however it ended.
+    # Listeners run on the executor's reaper thread.
+    def on_end(&block)
+      @on_end << block
+      self
+    end
+
+    def duration = started_at && finished_at ? finished_at - started_at : nil
+
+    # Block until the process has been reaped (or +limit+ seconds pass).
+    def wait(limit = nil)
+      @thread&.join(limit)
+      self
+    rescue StandardError # the reaper's own failure is already recorded as the state
+      self
+    end
+
+    # Captured stdout+stderr. With +tail+, only the last that many bytes.
+    def output(tail: nil)
+      return "" unless log_path && File.file?(log_path)
+
+      size = File.size(log_path)
+      text = if tail && size > tail
+               File.binread(log_path, tail, size - tail)
+             else
+               File.binread(log_path)
+             end
+      text.force_encoding("UTF-8").scrub
+    end
+
+    def output_size = (log_path && File.size?(log_path)) || 0
+
+    def to_h
+      {
+        id:, block_id:, step: step_slug, command:, background: background?, state: state.to_s, pid:,
+        started_at: started_at&.iso8601(3), finished_at: finished_at&.iso8601(3),
+        duration: duration&.round(3), exit_status:, signal:, error:,
+        cmd: cmd_path && File.basename(cmd_path), log: log_path && File.basename(log_path)
+      }
+    end
+
+    # --- transitions, called by Executor -----------------------------------
+
+    def started!(pid, at:)
+      @mutex.synchronize do
+        @pid        = pid
+        @started_at = at
+        @state      = :running
+      end
+      self
+    end
+
+    def attach(thread)
+      @thread = thread
+      self
+    end
+
+    # Ask the executor's reaper to end the process group. Returns true if
+    # there was something to stop.
+    def request_stop!
+      @mutex.synchronize do
+        return false unless running?
+
+        @stop = true
+      end
+    end
+
+    def finished!(status, at:, timed_out: false, stopped: false)
+      @mutex.synchronize do
+        @finished_at = at
+        @exit_status = status&.exitstatus || (status&.termsig && (128 + status.termsig))
+        @signal      = status&.termsig
+        @state       = if timed_out then :timed_out
+                       elsif stopped then :stopped
+                       else :finished
+                       end
+      end
+      ended!
+    end
+
+    def failed!(message, at:)
+      @mutex.synchronize do
+        @finished_at = at
+        @error       = message
+        @state       = :failed
+      end
+      ended!
+    end
+
+    private
+
+    # Each listener runs whatever the others do; one that raises is
+    # reported on stderr and skipped.
+    def ended!
+      @on_end.each do |listener|
+        listener.call(self)
+      rescue StandardError => e
+        warn "runsheets: execution #{id}: #{e.class}: #{e.message}"
+      end
+      self
+    end
+  end
+end

data/lib/runsheets/executor.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+module Runsheets
+  # Spawns block code as a child process group, pumps its stdout and stderr
+  # through the run's Redactor into the execution's log file, and reaps it
+  # in a background thread.
+  #
+  # Every execution is a fresh process: nothing carries over between blocks
+  # except the environment the session passes in.
+  class Executor
+    GRACE = 2.0   # seconds between TERM and KILL on timeout or stop
+    DRAIN = 1.0   # seconds to wait for the output pump after the process exits
+    POLL  = 0.05
+    CHUNK = 64 * 1024
+
+    attr_reader :clock
+
+    def initialize(clock: Time)
+      @clock = clock
+    end
+
+    # Write +code+ to the execution's cmd file and start it. Returns the
+    # execution, now running (or failed if it could not start). The
+    # execution's tee, when it has one, also receives the redacted output as
+    # it arrives and is closed at the end of the stream.
+    def start(execution, code:, env: {}, cwd: nil, redactor: nil)
+      File.write(execution.cmd_path, code)
+      log            = File.open(execution.log_path, "wb") # rubocop:disable Style/FileOpen -- closed by the pump thread
+      reader, writer = IO.pipe
+      pid = Process.spawn(
+        env.transform_keys(&:to_s).transform_values(&:to_s),
+        *execution.command, execution.cmd_path,
+        chdir: cwd || Dir.pwd, pgroup: true,
+        in: File::NULL, out: writer, err: writer
+      )
+      writer.close
+      pump = Thread.new { pump(reader, log, redactor || Redactor.new({}), execution.tee) }
+      execution.started!(pid, at: clock.now)
+      execution.attach(Thread.new { reap(execution, pid, pump) })
+    rescue SystemCallError, ArgumentError => e # ArgumentError: a NUL in the environment or command
+      writer&.close
+      reader&.close
+      log&.close
+      execution.tee&.close
+      execution.failed!("#{e.class}: #{e.message}", at: clock.now)
+    end
+
+    # Start and wait. Handy for scripts and tests.
+    def run(execution, **) = start(execution, **).wait
+
+    # Ask a running execution to end: TERM to its process group, then KILL.
+    # The reaper does the signalling; the execution ends up :stopped.
+    def stop(execution)
+      execution.request_stop!
+      execution
+    end
+
+    # Copy the child's output into the log, redacting as it goes. Runs until
+    # every holder of the pipe's write end has closed it.
+    def pump(io, log, redactor, tee = nil)
+      while (chunk = io.readpartial(CHUNK))
+        safe = redactor.feed(chunk)
+        log.write(safe)
+        log.flush
+        tee&.write(safe)
+      end
+    rescue IOError
+      nil
+    ensure
+      rest = redactor.flush
+      log.write(rest)
+      log.close
+      io.close
+      tee&.write(rest)
+      tee&.close
+    end
+
+    private
+
+    def reap(execution, pid, pump)
+      status, how = outcome(execution, pid)
+    rescue SystemCallError => e
+      execution.failed!("#{e.class}: #{e.message}", at: clock.now)
+    else
+      sweep_group(pid)
+      finish(execution, pump, status, **how)
+    end
+
+    # Wait for the process to end on its own, be stopped, or time out.
+    # Returns [status, how it ended].
+    def outcome(execution, pid)
+      deadline = execution.timeout && (monotonic + execution.timeout)
+      loop do
+        _, status = Process.wait2(pid, Process::WNOHANG)
+        return [status, {}] if status
+        return [end_group(pid), { stopped: true }] if execution.stop_requested?
+        return [end_group(pid), { timed_out: true }] if deadline && monotonic > deadline
+
+        sleep POLL
+      end
+    end
+
+    # Let the pump drain, then record how the process ended.
+    def finish(execution, pump, status, **how)
+      pump.join(DRAIN)
+      execution.finished!(status, at: clock.now, **how)
+    end
+
+    # Kill the group and return the child's status however it was reaped.
+    def end_group(pid) = kill_group(pid) || Process.wait2(pid).last
+
+    # TERM the whole process group, then KILL whatever is still there.
+    # Returns the child's status if it was reaped here, else nil.
+    def kill_group(pid)
+      signal(pid, "TERM")
+      stop = monotonic + GRACE
+      while monotonic < stop
+        reaped = Process.wait2(pid, Process::WNOHANG)
+        return reaped.last if reaped
+
+        sleep POLL
+      end
+      signal(pid, "KILL")
+      nil
+    end
+
+    # The block's own process has been reaped; end anything it left behind
+    # in its group (`cmd &`, `nohup cmd &`) the same way: TERM, then KILL
+    # after GRACE. A block that started nothing returns at once.
+    def sweep_group(pid)
+      %w[TERM KILL].each do |name|
+        break unless group_alive?(pid)
+
+        signal(pid, name)
+        stop = monotonic + GRACE
+        sleep POLL while group_alive?(pid) && monotonic < stop
+      end
+    end
+
+    def group_alive?(pid)
+      Process.kill(0, -pid)
+      true
+    rescue Errno::ESRCH
+      false
+    rescue Errno::EPERM
+      true
+    end
+
+    def signal(pid, name)
+      Process.kill(name, -pid)
+    rescue Errno::ESRCH, Errno::EPERM
+      nil
+    end
+
+    def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  end
+end

data/lib/runsheets/expectation.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Runsheets
+  # Compares a block's output with the text of its expect block. Lines are
+  # matched in order with trailing whitespace ignored, and `{{any}}` in the
+  # expected text matches any characters within one line. The result lines
+  # the two up row by row for the page, with the characters that differ
+  # marked, so line N of the output sits beside line N of what was expected.
+  module Expectation
+    ANY  = "{{any}}"
+    DIFF = "diff"
+    WILD = "any"
+
+    # One row of the side-by-side view. +tag+ is :same, :changed, :expected
+    # (a line only the expect block has) or :actual (only the output has).
+    # Each side is a list of [text, mark] segments, mark nil, "diff" or
+    # "any"; a side is nil when the row has no line there.
+    Row = Data.define(:tag, :expected, :actual) do
+      def to_h = { tag: tag.to_s, expected:, actual: }
+    end
+
+    Result = Data.define(:rows) do
+      def match? = rows.all? { it.tag == :same }
+      def to_h   = { match: match?, rows: rows.map(&:to_h) }
+    end
+
+    def self.compare(expected, actual)
+      patterns = Hash.new { |cache, line| cache[line] = pattern(line) }
+      Result.new(rows(ops(lines(expected), lines(actual), ->(want, got) { patterns[want].match?(got) })))
+    end
+
+    # The edit script, with the lines that match at the start and at the
+    # end taken out before the diff, whose cost grows with the product of
+    # the two line counts. Output that mostly matches leaves it little to do.
+    def self.ops(wants, gots, same)
+      head = matching_run(wants, gots, same)
+      tail = matching_run(wants.drop(head).reverse, gots.drop(head).reverse, same)
+      middle = Diff.ops(wants[head...(wants.size - tail)], gots[head...(gots.size - tail)], same:)
+      paired(wants.first(head), gots.first(head)) + middle + paired(wants.last(tail), gots.last(tail))
+    end
+
+    # How many lines from the start match, pair by pair.
+    def self.matching_run(wants, gots, same) = wants.zip(gots).take_while { |want, got| got && same.call(want, got) }.size
+
+    def self.paired(wants, gots) = wants.zip(gots).map { |want, got| [" ", want, got] }
+
+    # The comparison as a page wants it, or nil when there is nothing to
+    # compare: no expect block, or an execution still running or that never
+    # started.
+    def self.for_page(expected, output, state)
+      return nil if expected.nil? || %w[running failed].include?(state.to_s)
+
+      compare(expected, output).to_h
+    end
+
+    def self.lines(text) = text.to_s.gsub(/[ \t]+$/, "").rstrip.lines(chomp: true)
+
+    # A line of expected text as a whole-line pattern.
+    def self.pattern(line) = /\A#{line.split(ANY, -1).map { Regexp.escape(it) }.join('.*')}\z/
+
+    # Rows from the edit script: unchanged lines as they are, and each run of
+    # changed lines paired off expected against actual.
+    def self.rows(ops)
+      ops.chunk_while { |a, b| (a.first == " ") == (b.first == " ") }.flat_map do |chunk|
+        chunk.first.first == " " ? same_rows(chunk) : changed_rows(chunk.filter_map { it[1] }, chunk.filter_map { it[2] })
+      end
+    end
+
+    def self.same_rows(chunk) = chunk.map { |_, want, got| Row.new(:same, wildcards(want), plain(got)) }
+
+    def self.changed_rows(wants, gots)
+      Array.new([wants.size, gots.size].max) { row(wants[it], gots[it]) }
+    end
+
+    def self.row(want, got)
+      return Row.new(:expected, wildcards(want, DIFF), nil) if got.nil?
+      return Row.new(:actual, nil, plain(got, DIFF)) if want.nil?
+      return Row.new(:changed, wildcards(want, DIFF), plain(got, DIFF)) if want.include?(ANY)
+
+      prefix = common_prefix(want, got)
+      suffix = common_prefix(want[prefix..].reverse, got[prefix..].reverse)
+      Row.new(:changed, around(want, prefix, suffix), around(got, prefix, suffix))
+    end
+
+    # +text+ in segments, with each {{any}} marked as a wildcard.
+    def self.wildcards(text, mark = nil)
+      text.split(/(#{Regexp.escape(ANY)})/).reject(&:empty?).map { [it, it == ANY ? WILD : mark] }
+    end
+
+    def self.plain(text, mark = nil) = text.empty? ? [] : [[text, mark]]
+
+    # +text+ with the part between its first +prefix+ and last +suffix+
+    # characters marked as the difference.
+    def self.around(text, prefix, suffix)
+      [[text[0, prefix], nil], [text[prefix...(text.size - suffix)], DIFF], [text[(text.size - suffix)..], nil]]
+        .reject { it.first.empty? }
+    end
+
+    def self.common_prefix(one, other) = one.each_char.zip(other.each_char).take_while { |mine, theirs| mine == theirs }.size
+  end
+end