openlanes 3.0.0

data/lib/openlanes/engine/progress_log.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Engine
+    # One line of `run --log FILE`, laid out as fixed-width columns.
+    #
+    # The log file is the only record of a headless run, and it is shared: two
+    # `openlanes run` invocations pointed at the same `--log` both append to
+    # it, interleaved, and so does every round within one run. That is what the
+    # process id column is for. Without it a reader cannot tell one run's agents
+    # from another's, and "started 003.00" twice looks like a bug rather than two
+    # machines doing the same work.
+    #
+    # Fixed columns are what make the file readable by eye and by `awk` alike. A
+    # value wider than its column is cut rather than allowed to push the columns
+    # out, and a value that is missing still occupies its width, so a round header
+    # with no plan or agent of its own lines up with the lines underneath it.
+    #
+    # @example
+    #   ProgressLog.render("editing spec.md", plan: "003.00", status: "⭐️ Planned",
+    #     agent: "yoda-writer", seconds: 42, round: "01", pid: 91_234)
+
+    #   #=> "[16:22:14 | 003.00 | ⭐️ Planned            | yoda-writer          | 01 |   91234 |    42s] editing spec.md"
+    module ProgressLog
+      # Wall clock only. A log line answers "when", and the date is the file's.
+      TIME_FORMAT = "%H:%M:%S"
+
+      # Width of {TIME_FORMAT}'s output.
+      TIME_WIDTH = 8
+
+      # "003.00".
+      PLAN_WIDTH = 6
+
+      # Derived from the widest state as {Status#to_s} renders it, emoji
+      # included, so a state added to {STATUSES} with a longer label cannot
+      # silently knock the columns after it out of line — then trimmed by five
+      # cells: the emoji already names the state, so the longest labels can
+      # afford to lose their tails to keep the line short.
+      STATUS_WIDTH = Plans::STATUSES.map { |status| UI.display_width(status.to_s) }.max - 5
+
+      # The longest specialist name in `agents/` is `palpatine-planner` at 17,
+      # and the names are hyphenated words rather than a bounded vocabulary, so
+      # this leaves room for one more without a reflow.
+      AGENT_WIDTH = 20
+
+      # Enough for a 32-bit process id, right justified.
+      PID_WIDTH = 7
+
+      # "01". Rounds cap at two digits; a run that reaches a third has bigger
+      # problems than this column.
+      ROUND_WIDTH = 2
+
+      # "  907s". Anything longer than four digits of seconds is an agent nobody
+      # is still waiting on.
+      SECONDS_WIDTH = 6
+
+      # Between columns, inside the brackets.
+      SEPARATOR = " | "
+
+      # Cells from the opening bracket to the closing one, separators included.
+      # The message starts one space after this on every line, whatever fields
+      # that line happens to be missing, so a reader indenting a wrapped message
+      # has a number to indent by.
+      COLUMN_WIDTHS = [TIME_WIDTH, PLAN_WIDTH, STATUS_WIDTH, AGENT_WIDTH, ROUND_WIDTH, PID_WIDTH,
+                       SECONDS_WIDTH].freeze
+
+      # @see COLUMN_WIDTHS
+      LINE_WIDTH = COLUMN_WIDTHS.sum + (SEPARATOR.length * (COLUMN_WIDTHS.size - 1)) + 2
+
+      class << self
+        # Render one log line: bracketed fixed-width columns, then the message.
+        #
+        # No colour. The line goes to a file, where an escape sequence is neither
+        # readable nor the width it claims to be.
+        #
+        # @param message [String] the free-form tail, the only variable-width part
+        # @param plan [String, nil] the plan's ordinal, e.g. "003.00"
+        # @param status [String, nil] emoji and label, e.g. "⭐️ Planned"
+        # @param agent [String, nil] the specialist's name, e.g. "yoda-writer"
+        # @param seconds [Numeric, nil] how long this agent has been alive
+        # @param round [String, nil] which pass over the tree, e.g. "01"
+        # @param pid [Integer, nil] which run wrote the line
+        # @param at [Time] injectable, so a caller can render a fixed clock
+        # @return [String] one line, with no trailing newline
+        def render(message, plan: nil, status: nil, agent: nil, seconds: nil,
+          round: nil, pid: Process.pid, at: Time.now)
+          columns = [
+            left(at.strftime(TIME_FORMAT), TIME_WIDTH),
+            left(plan, PLAN_WIDTH),
+            left(status, STATUS_WIDTH),
+            left(agent, AGENT_WIDTH),
+            right(round, ROUND_WIDTH),
+            right(pid, PID_WIDTH),
+            right(duration(seconds), SECONDS_WIDTH)
+          ]
+
+          text = message.to_s
+          prefix = "[#{columns.join(SEPARATOR)}]"
+          text.empty? ? prefix : "#{prefix} #{text}"
+        end
+
+        # @param seconds [Numeric, nil]
+        # @return [String] whole seconds with a trailing "s", or blank for nil
+        def duration(seconds) = seconds.nil? ? "" : "#{seconds.round}s"
+
+        # @param value [Object, nil]
+        # @param width [Integer] terminal cells
+        # @return [String] padded on the right to exactly +width+ cells
+        def left(value, width) = UI.fit(value, width)
+
+        # {UI.fit} pads on the right, which is the wrong end for a number. Fit
+        # first so an over-long value is still truncated to the column, then move
+        # the padding to the front.
+        #
+        # @param value [Object, nil]
+        # @param width [Integer] terminal cells
+        # @return [String] padded on the left to exactly +width+ cells
+        def right(value, width)
+          text = UI.fit(value, width).rstrip
+          UI.fit("", width - UI.display_width(text)) + text
+        end
+      end
+    end
+  end
+end

data/lib/openlanes/engine/runner.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Engine
+    # Drives specialist agents over a `.plans` tree until nothing is left to
+    # dispatch.
+    #
+    # The loop itself is {Dispatcher}. This holds what the loop needs about
+    # the run: the tree and its scope, the roster, the executor, how many run
+    # at once, where each plan is checked out, and how a finished plan is
+    # published. Blocked plans (⭕️ 🅱️) are stepped around, never assigned.
+    class Runner
+      # What one agent did to one plan.
+      Attempt = Data.define(:ordinal,
+        :agent,
+        :from,
+        :to,
+        :ok,
+        :note,
+        :up,
+        :down,
+        :subagents,
+        :delegated,
+        :seconds,
+        :round,
+        :file,
+        :model,
+        :status) do
+        def initialize(round: 1, file: "", model: nil, status: nil, **rest) = super
+
+        # @return [Boolean] whether the plan actually moved
+        def advanced? = ok && from != to
+      end
+
+      # One unit of work: an agent, a plan, and the checkout it happens in.
+      Task = Data.define(:agent, :subject, :root, :checkout, :round) do
+        # @return [Hash] columns for the log
+        def log_fields
+          { plan: subject.feature.ordinal.to_s, status: subject.status.to_s,
+           agent: agent.name, round: format("%02d", round) }
+        end
+      end
+
+      # @param rounds [Integer, nil] `--rounds`, capping each agent's own
+      # @param state [OpenLanes::Engine::StateFile, nil]
+      # @param sleeper [Proc]
+      # @param on_board [Proc, nil]
+      # @param claims [OpenLanes::Live::Claims, nil] plan claims shared with
+      #   other machines; nil when the run is not live, and every plan is ours
+      # @param stay [Boolean] idle past fixed points until a person stops
+      #   the run; see {Dispatcher#done?}
+      # @param rescan [Integer] seconds between rescans while idle
+      def initialize(tree:, executor:, agents: Agents::Registry.new, isolation: :shared, jobs: 1, worktree: nil,
+        plans: nil, publisher: nil, dry_run: false, rounds: nil, state: nil,
+        sleeper: ->(seconds) { sleep(seconds) }, on_board: nil, claims: nil, stay: false, rescan: Dispatcher::RESCAN)
+        @tree = tree
+        @executor = executor
+        @agents = agents
+        @isolation = isolation
+        @worktree = worktree
+        @plans = plans
+        @publisher = publisher
+        @dry_run = dry_run
+        @rounds = rounds
+        @state = dry_run ? nil : state
+        @sleeper = sleeper
+        @on_board = on_board
+        @claims = claims
+        @stay = stay
+        @rescan = rescan
+        @attempts = []
+        @dropped = []
+        @jobs = isolated? ? jobs : 1
+      end
+
+      attr_reader :jobs, :worktree, :tree, :attempts, :executor, :agents
+
+      # @return [Boolean]
+      def isolated? = @isolation == :worktree
+
+      # @return [Boolean]
+      def dry_run? = @dry_run
+
+      # Whether a finished build is pushed and its pull request opened. Not
+      # with `--isolation shared` or `--no-git-push`, the local-only runs, where
+      # a plan can never hold the open pull request 🟢 requires.
+      #
+      # @return [Boolean]
+      def publishes? = !@publisher.nil?
+
+      # @return [String] the repository root
+      def root = isolated? ? worktree.root : shared_root
+
+      # @param subject [OpenLanes::Plans::Subject]
+      # @return [Boolean]
+      def in_scope?(subject)
+        ordinal = subject.feature.ordinal
+        !@dropped.include?(ordinal.to_s) && (@plans.nil? || @plans.include?(ordinal))
+      end
+
+      # Take a plan out of this run, as when another machine now holds it.
+      #
+      # @param ordinal [#to_s]
+      # @return [void]
+      def drop_plan(ordinal) = @dropped |= [ordinal.to_s]
+
+      # @return [Array<OpenLanes::Plans::Subject>]
+      def in_scope = tree.subjects.select { |s| in_scope?(s) }
+
+      # Widen a run scoped by `--plan` to one more plan, from the page's
+      # "Choose an existing plan". An unscoped run already has every plan.
+      #
+      # @param ordinal [OpenLanes::Plans::Ordinal]
+      # @return [Boolean] whether the scope grew
+      def add_plan(ordinal)
+        @dropped.delete(ordinal.to_s)
+        return false if @plans.nil? || @plans.include?(ordinal)
+
+        @plans << ordinal
+        true
+      end
+
+      # Whether another machine holds the plan (W26). Such a plan is stepped
+      # around, as a blocked one is.
+      #
+      # @param subject [OpenLanes::Plans::Subject]
+      # @return [Boolean]
+      def foreign?(subject)
+        !@claims.nil? && @claims.foreign?(subject.feature.ordinal)
+      rescue Live::Unavailable => e
+        claims_unavailable(e)
+        false
+      end
+
+      # Take the plan before an agent starts on it. With Redis gone the run
+      # carries on unclaimed rather than stalling: the claim is an agreement
+      # between machines, and a machine that cannot reach the others is
+      # working alone.
+      #
+      # @param subject [OpenLanes::Plans::Subject]
+      # @return [Boolean] whether this machine may work it
+      def claim(subject)
+        @claims.nil? || @claims.claim(subject.feature.ordinal)
+      rescue Live::Unavailable => e
+        claims_unavailable(e)
+        true
+      end
+
+      # Run until nothing is dispatchable, or, staying, until a person stops
+      # it.
+      #
+      # @yieldparam dispatcher [OpenLanes::Engine::Dispatcher] before it runs, so a
+      #   console can attach itself and reach the running jobs
+      # @return [Array<OpenLanes::Engine::Runner::Attempt>]
+      def call
+        dispatcher = Dispatcher.new(runner: self,
+          state:    @state,
+          rounds:   @rounds,
+          sleeper:  @sleeper,
+          on_board: @on_board,
+          stay:     @stay,
+          rescan:   @rescan)
+        yield dispatcher if block_given?
+        @attempts = dispatcher.run
+      end
+
+      # @return [Array<OpenLanes::Plans::Subject>]
+      def blocked = in_scope.select { |s| %i[blocked product_blocked].include?(s.status.key) }
+
+      # @return [Boolean]
+      def settled? = in_scope.all? { |s| Plans::StateMachine::SETTLED.include?(s.status.key) }
+
+      # A dry run gets the shared root whatever the isolation says: nothing
+      # runs in the checkout, and `git worktree add` is a side effect a
+      # preview must not have. It also fails outright when the plan's branch
+      # is the one checked out where the run was typed, which is where a dry
+      # run of that plan is typed.
+      #
+      # @return [OpenLanes::Engine::Runner::Task]
+      def prepare(agent, subject, round)
+        return Task.new(agent:, subject:, root: shared_root, checkout: nil, round:) if !isolated? || dry_run?
+
+        checkout = worktree.checkout_for(subject.feature)
+        Task.new(agent:, subject:, root: checkout.path, checkout:, round:)
+      end
+
+      # Push a finished build and record the pull request it opened in the
+      # plan's `pull-requests.md`, the row 🟢 is judged by. A push to a pull
+      # request the plan already had records nothing: the row is there, and a
+      # second copy of it would read as a second pull request.
+      #
+      # @param task [OpenLanes::Engine::Runner::Task]
+      # @param subject [OpenLanes::Plans::Subject] the plan in the main tree
+      # @return [OpenLanes::Vcs::Publisher::Publication, nil] nil when the run
+      #   publishes nothing or the checkout holds no change
+      def publish(task, subject)
+        return nil unless @publisher && task.checkout&.dirty?
+
+        publication = @publisher.publish(checkout: task.checkout, subject:)
+        record_pull_request(subject.feature.path, publication) if publication.published? && !publication.updated
+        publication
+      end
+
+      private
+
+      # @return [String]
+      def shared_root = File.dirname(tree.dir)
+
+      # @param error [OpenLanes::Live::Unavailable]
+      # @return [void]
+      def claims_unavailable(error) = UI.log("claims unchecked: #{error.message}")
+
+      # @return [void]
+      def record_pull_request(path, publication)
+        number = publication.url.to_s[%r{/pull/(\d+)}, 1]
+        rows = Plans::PullRequests.new(dir: path).all.map { |pr|
+          { number: pr.number, title: pr.title, url: pr.url, state: pr.state, body: "" }
+        }
+        rows << { number:, title: publication.title, url: publication.url, state: "Open 🟡", body: "" }
+        Plans::PullRequests.upsert(File.join(path, Plans::PullRequests::FILENAME), rows)
+      end
+    end
+  end
+end

data/lib/openlanes/engine/state_file.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+
+module OpenLanes
+  module Engine
+    # Where a run stands, written every second, so a harness that dies leaves
+    # the next one something to restart from.
+    #
+    # The ledger inside each document is the record humans read and the
+    # dispatcher acts on; this is the index of it, plus what the ledger cannot
+    # hold: which run wrote what, process ids, tokens, and whether the run
+    # that wrote a `Started` is still alive. It sits in `.plans/` rather than
+    # the system temp dir because a reboot must not lose it, and it is
+    # gitignored because it is about one machine's run, not the plan.
+    class StateFile
+      FILENAME = "openlanes-state.json"
+
+      # Where it used to live, one directory down. Read by {#load} when the
+      # current path holds nothing, so a run started under the old layout is
+      # picked up rather than silently restarted from zero. Nothing writes
+      # here any more and nothing deletes it: the file left behind is the
+      # only record a half-finished run has.
+      LEGACY_DIRNAME = "tmp"
+
+      # The lines {.ensure_ignored!} adds. The second covers the sibling
+      # {#save} writes and renames over the first — brief, but `.plans/` is a
+      # directory whose other contents are committed, so a `git status` run
+      # during that instant must not offer it.
+      IGNORE = ["#{OpenLanes::PLANS_DIR}/#{FILENAME}", "#{OpenLanes::PLANS_DIR}/#{FILENAME}.*.tmp"].freeze
+
+      # @param tree [OpenLanes::Plans::Tree]
+      # @return [String]
+      def self.for(tree) = File.join(tree.dir, FILENAME)
+
+      # @param tree [OpenLanes::Plans::Tree]
+      # @return [String] where {LEGACY_DIRNAME} put it
+      def self.legacy_for(tree) = File.join(tree.dir, LEGACY_DIRNAME, FILENAME)
+
+      # Add the ignore lines once. Editing somebody's `.gitignore` is a thing
+      # to announce, which is why this returns whether it did.
+      #
+      # Only the first line is asked about: git answers for a literal path,
+      # and the second is a glob no path equals. They are written together,
+      # so one being ignored means both are.
+      #
+      # @param root [String] repository root
+      # @return [Boolean] true when lines were added
+      def self.ensure_ignored!(root)
+        return false unless system("git", "-C", root, "rev-parse", "--git-dir", out: File::NULL, err: File::NULL)
+        return false if system("git", "-C", root, "check-ignore", "-q", IGNORE.first, out: File::NULL, err: File::NULL)
+
+        path = File.join(root, ".gitignore")
+        existing = File.file?(path) ? File.read(path) : ""
+        glue = existing.empty? || existing.end_with?("\n") ? "" : "\n"
+        File.write(path, "#{existing}#{glue}#{IGNORE.join("\n")}\n")
+        true
+      end
+
+      # @param path [String]
+      # @param pid [Integer] this run's process id
+      def initialize(path:, pid: Process.pid)
+        @path = path
+        @pid = pid
+        @data = { "run" => {}, "plans" => {} }
+        @mutex = Mutex.new
+      end
+
+      # @return [String]
+      attr_reader :path
+
+      # Where a run started before the file moved up out of {LEGACY_DIRNAME}
+      # would have written. {#load} falls back to it; {#save} never uses it.
+      #
+      # @return [String]
+      def legacy_path = File.join(File.dirname(path), LEGACY_DIRNAME, File.basename(path))
+
+      # Reads the current path, or the legacy one when the current path holds
+      # nothing. A run interrupted under the old layout is otherwise invisible
+      # to the run that picks it up, and {#stranded} would report no stranded
+      # stages for a harness that died with several.
+      #
+      # @return [self]
+      def load
+        @mutex.synchronize do
+          source = [path, legacy_path].find { |candidate| File.file?(candidate) }
+          @data = JSON.parse(File.read(source)) if source
+          @data = { "run" => {}, "plans" => {} } unless @data.is_a?(Hash) && @data["plans"].is_a?(Hash)
+        end
+        self
+      rescue JSON::ParserError
+        self
+      end
+
+      # Written whole to a sibling and renamed over, so a reader never sees
+      # half a file.
+      #
+      # @return [void]
+      def save
+        @mutex.synchronize do
+          FileUtils.mkdir_p(File.dirname(path))
+          temp = "#{path}.#{@pid}.tmp"
+          File.write(temp, JSON.pretty_generate(@data))
+          File.rename(temp, path)
+        end
+      end
+
+      # @param root [String]
+      # @return [void]
+      def begin_run!(root:)
+        @mutex.synchronize do
+          @data["run"] = { "pid" => @pid, "root" => root, "started_at" => Time.now.iso8601,
+                          "heartbeat_at" => Time.now.iso8601 }
+        end
+      end
+
+      # @return [void]
+      def heartbeat!
+        @mutex.synchronize { @data["run"]["heartbeat_at"] = Time.now.iso8601 }
+      end
+
+      # @param ordinal [String]
+      # @param agent [String]
+      # @param round [Integer]
+      # @param fields [Hash] anything else worth keeping: status, model, file,
+      #   pid, up, down, state, next, exit
+      # @return [void]
+      def record(ordinal, agent:, round:, **fields)
+        @mutex.synchronize do
+          plan = (@data["plans"][ordinal.to_s] ||= { "stages" => [] })
+          stage = plan["stages"].find { |s| s["agent"] == agent && s["round"] == round }
+          unless stage
+            stage = { "agent" => agent, "round" => round, "run_pid" => @pid }
+            plan["stages"] << stage
+          end
+          fields.each { |key, value| stage[key.to_s] = value }
+        end
+      end
+
+      # The live run's record (W6): what a person said about the run that no
+      # plan folder holds, kept here as the copy of Redis that survives Redis
+      # losing it. {Live::RunRecord} writes it and reads it back on start;
+      # it carries its own `seq`, so the newer of the two copies wins.
+      #
+      # @return [Hash, nil]
+      def live = @mutex.synchronize { @data["live"]&.dup }
+
+      # @param data [Hash] written by the next {#save}
+      # @return [void]
+      def live=(data)
+        @mutex.synchronize { @data["live"] = data }
+      end
+
+      # @param ordinal [String]
+      # @return [Array<Hash>]
+      def stages(ordinal) = @mutex.synchronize { (@data.dig("plans", ordinal.to_s, "stages") || []).map(&:dup) }
+
+      # @return [Array<String>] every ordinal with a stage recorded
+      def plans = @mutex.synchronize { @data["plans"].keys }
+
+      # @return [Boolean] whether the run that last wrote this file is gone
+      def previous_run_dead?
+        previous = @data.dig("run", "pid")
+        return false if previous.nil? || previous == @pid
+
+        Process.kill(0, previous)
+        false
+      rescue Errno::ESRCH, Errno::EPERM
+        true
+      end
+
+      # Stages a dead run left `Started`. Each is an agent that was cut off
+      # mid-flight and has to be re-run, with the harness's own Interrupted
+      # line written so the document agrees with this file.
+      #
+      # @return [Array<Array(String, Hash)>] ordinal and stage
+      def stranded
+        return [] unless previous_run_dead?
+
+        @mutex.synchronize do
+          @data["plans"].flat_map { |ordinal, plan|
+            plan["stages"].select { |s| s["status"] == "Started" && s["run_pid"] != @pid }
+                          .map { |s| [ordinal, s.dup] }
+          }
+        end
+      end
+    end
+  end
+end