openlanes 3.0.0 → 3.1.0

data/lib/openlanes/engine/agent_locks.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+require "json"
+require "socket"
+require "tty/command"
+
+module OpenLanes
+  module Engine
+    # The plans an agent outside openlanes is working on, as `alock` (the
+    # agent-lock gem) records them, and the engine's own claim in the same
+    # register (decision W31).
+    #
+    # The Redis claim (W26) keeps engines apart and the lock file keeps two
+    # runs on one machine apart, but an agent a person started by hand reads
+    # neither. It does read alock: every agent here claims what it is about
+    # to write. So the engine stays out of a plan whose folder, or whose
+    # worktree, somebody else holds, and holds the folder itself while its
+    # agents are in it, so that agent sees the engine too. With alock on
+    # Redis this reaches across machines as well.
+    #
+    # `alock acquire` is the atomic step: it refuses (exit 1) when anybody
+    # holds an overlapping scope, so two contenders cannot both win. The
+    # listing only keeps the engine from asking, tick after tick, for plans
+    # it would be refused.
+    #
+    # alock is a command, not a library: its holder is the process that runs
+    # it, skipping shells, which is the engine itself, so a lock ends with
+    # the engine that took it. A machine without alock has no agent using it
+    # either, and the engine carries on without it, said once.
+    class AgentLocks
+      # How long one `alock list` answers for. The dispatcher asks on every
+      # one-second tick, and each answer starts a Ruby process.
+      FRESH = 5
+
+      # One lock, as `alock list --json` reports it.
+      #
+      # @!attribute [r] agent
+      #   @return [String] the holder's agent id
+      # @!attribute [r] scope
+      #   @return [String] relative to {#tree}, e.g. `.plans/003.00━━🟡━━❯x/**`
+      # @!attribute [r] tree
+      #   @return [String] the checkout the lock was taken in
+      # @!attribute [r] intent
+      #   @return [String, nil]
+      Held = Data.define(:agent, :scope, :tree, :intent) do
+        # @return [String] the scope's literal part: `**` is "", `a/b/**` is "a/b"
+        def fixed = scope.split("/").take_while { !it.match?(/[*?\[{]/) }.join("/")
+
+        # @return [String] who holds it and why, for the log
+        def to_s = "#{agent} (#{intent || "no intent given"})"
+      end
+
+      # @param root [String] the repository whose plans are locked
+      # @param id [String] the agent id the engine's own locks carry
+      # @param own [Array<String>] the engine's agents, who claim with
+      #   `AGENT_ID=<name>` or `<name>-<suffix>` as their prompt tells them to
+      # @param command [TTY::Command]
+      # @param clock [#call] monotonic seconds
+      def initialize(root:, id: "openlanes@#{Socket.gethostname}:#{Process.pid}", own: [],
+        command: TTY::Command.new(printer: :null), clock: -> { UI.monotonic })
+        @root = File.realpath(root)
+        @id = id
+        @own = own
+        @failing = nil
+        @command = command
+        @clock = clock
+        @mutex = Mutex.new
+        @snapshot = nil
+        @taken_at = nil
+        @off = false
+      end
+
+      # @return [String]
+      attr_reader :id
+
+      # Somebody else's live lock that overlaps the plan's folder, or any
+      # lock taken inside its worktree. The engine's own agents are not
+      # somebody else: they claim each file before writing it, so counting
+      # them would keep a pair's partner out of the plan while the first one
+      # works in it.
+      #
+      # @param dir [String] the plan folder
+      # @param checkout [String, nil] the plan's worktree, when the run isolates
+      # @return [Held, nil]
+      def held_elsewhere(dir, checkout: nil)
+        folder = real(dir)
+        work = checkout && real(checkout)
+        held.find do |lock|
+          next false if ours?(lock.agent)
+
+          tree = real(lock.tree)
+          (work && tree == work) || (inside?(folder, tree) && overlap?(relative(folder, tree), lock.fixed))
+        end
+      end
+
+      # Anybody's live lock taken inside one checkout, the engine's own
+      # included: a worktree with a lock in it is in use, whoever holds it.
+      #
+      # @param checkout [String]
+      # @return [Held, nil]
+      def held_in(checkout)
+        work = real(checkout)
+        held.find { real(it.tree) == work }
+      end
+
+      # Claim the plan's folder. False when somebody holds it, in which case
+      # the plan is left alone; true when alock is not installed.
+      #
+      # @param dir [String] the plan folder
+      # @param intent [String] what the engine is about to do there
+      # @return [Boolean]
+      def acquire(dir, intent)
+        run("acquire", scope(dir), intent, dir: File.dirname(dir))
+      end
+
+      # @param dir [String] the plan folder, as it was named when acquired
+      # @return [void]
+      def release(dir)
+        run("release", scope(dir), dir: File.dirname(dir))
+      end
+
+      private
+
+      # @param agent [String] a lock's holder
+      # @return [Boolean] the engine itself, one of its agents, or a sub-agent of one
+      def ours?(agent) = agent == @id || @own.any? { agent == it || agent.start_with?("#{it}-") }
+
+      # @return [Array<Held>] every active lock, at most {FRESH} seconds old
+      def held
+        @mutex.synchronize do
+          return [] if @off
+          return @snapshot if @snapshot && @clock.call - @taken_at < FRESH
+
+          @snapshot = list
+          @taken_at = @clock.call
+          @snapshot
+        end
+      end
+
+      # @return [Array<Held>]
+      def list
+        out = @command.run("alock", "list", "--json", "--dir", @root).out
+        listed = JSON.parse(out).filter_map do |lock|
+          next unless lock["status"] == "active"
+
+          Held.new(agent: lock["agent_id"].to_s, scope: lock["scope"].to_s, tree: lock["tree"].to_s, intent: lock["intent"])
+        end
+        @failing = nil
+        listed
+      rescue TTY::Command::ExitError, JSON::ParserError => e
+        failing(e)
+        []
+      rescue SystemCallError
+        missing
+        []
+      end
+
+      # @return [Boolean] exit 0; false on exit 1, which is a refusal
+      def run(*args, dir:)
+        return true if @off
+
+        result = @command.run!("alock", *args, "--dir", dir, env: { "AGENT_ID" => @id })
+        @mutex.synchronize { @snapshot = nil }
+        # 1 is a refusal; anything above is alock failing, which is said, and
+        # counts as a refusal too: a plan nobody could check is left alone.
+        if result.exit_status > 1
+          left = args.first == "release" ? "; #{args[1]} stays held until the engine exits" : ""
+          UI.log("alock #{args.first} #{args[1]} failed: #{(result.err.to_s + result.out.to_s).lines.first.to_s.strip}#{left}",
+            level: :warn)
+        end
+        result.success?
+      rescue SystemCallError
+        missing
+        true
+      end
+
+      # Said once per failure, not on every tick it lasts: the listing is
+      # asked for every few seconds, and the same line each time buried the
+      # log. A listing that works again clears it.
+      #
+      # @param error [Exception]
+      # @return [void]
+      def failing(error)
+        # TTY::Command's message is the command line first; the reason is in
+        # the stderr lines after it.
+        reason = error.message.lines.map(&:strip).reject(&:empty?).find { it.start_with?("stderr:") } ||
+                 error.message.lines.first.to_s.strip
+        return if @failing == reason
+
+        @failing = reason
+        UI.log("alock list failed, plans unchecked until it works: #{reason}", level: :warn)
+      end
+
+      # @return [void]
+      def missing
+        return if @off
+
+        @off = true
+        UI.log("alock is not installed: plans are not checked against agents outside openlanes", level: :warn)
+      end
+
+      # The folder and everything in it: a directory scope covers its
+      # contents, and the folder's own files are what the agents write.
+      #
+      # @param dir [String]
+      # @return [String] relative to the folder's parent, where alock runs
+      def scope(dir) = "#{File.basename(dir)}/**"
+
+      # @param path [String]
+      # @return [String]
+      def real(path) = File.exist?(path) ? File.realpath(path) : File.expand_path(path)
+
+      # @return [Boolean]
+      def inside?(path, tree) = path == tree || path.start_with?("#{tree}/")
+
+      # @return [String]
+      def relative(path, tree) = path == tree ? "" : path.delete_prefix("#{tree}/")
+
+      # alock's own rule: two scopes conflict when either one's literal part
+      # contains the other's, and "" (the whole tree) contains everything.
+      #
+      # @param path [String]
+      # @param fixed [String]
+      # @return [Boolean]
+      def overlap?(path, fixed)
+        fixed.empty? || path.empty? || path == fixed ||
+          path.start_with?("#{fixed}/") || fixed.start_with?("#{path}/")
+      end
+    end
+  end
+end

data/lib/openlanes/engine/arbitration.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Engine
+    # Where a plan goes once its rejections reach `merge.max_rejections`
+    # (M6): to the arbitrator, `obi-wan-kenobi-arbitrator` (M7, M8), which
+    # plan 015 builds.
+    #
+    # Until then this is the seam it plugs into, and its default only says
+    # so. The plan is handled as any other rejection, so nothing waits on a
+    # ruling nobody can give yet; holding it under Needs you at the cap is
+    # plan 014.02's.
+    class Arbitration
+      # @param plan [OpenLanes::Plans::Subject]
+      # @param count [Integer] rejections so far, hansolo's and chewie's
+      # @param max [Integer] `merge.max_rejections`
+      # @return [void]
+      def call(plan:, count:, max:)
+        UI.log("rejected #{count} of #{max}: at the cap, and no arbitrator is built yet (plan 015)",
+          level: :warn,
+          plan:  plan.feature.ordinal.to_s)
+      end
+    end
+  end
+end

data/lib/openlanes/engine/board.rb

@@ -3,12 +3,18 @@
 module OpenLanes
   module Engine
     # What the screen draws: everything about the run, as values, refreshed
-    # once a second by the dispatcher. The screen paints it and knows nothing
+    # every tick by the dispatcher. The screen paints it and knows nothing
     # about threads; the dispatcher builds it and knows nothing about cells.
     #
     # @!attribute [r] rows
     #   @return [Array<OpenLanes::Engine::Board::Row>] one per agent invocation still
     #     worth showing: running, or finished within the last few seconds
+    # @!attribute [r] waiting_on
+    #   @return [Hash{String => Array<Integer>}] plan to the pull requests open
+    #     and waiting on a person, as the human-mode watch last saw them (M4)
+    # @!attribute [r] jobs
+    #   @return [Integer] how many agents may run at once now, after any
+    #     `/concurrency` from the copilot (plan 020.00)
     Board = Data.define(:started_at,
       :status,
       :plans,
@@ -23,8 +29,10 @@ module OpenLanes
       :dialog,
       :help,
       :about,
-      :hidden) do
-      def initialize(selected: nil, dialog: nil, help: false, about: nil, hidden: 0, **rest) = super
+      :hidden,
+      :waiting_on,
+      :jobs) do
+      def initialize(selected: nil, dialog: nil, help: false, about: nil, hidden: 0, waiting_on: {}, jobs: 1, **rest) = super
     end
 
     # @!attribute [r] key
@@ -52,6 +60,13 @@ module OpenLanes
     # @!attribute [r] session
     #   @return [String, nil] the CLI's session id, once its stream named it;
     #     what a compaction resumes (W28)
+    # @!attribute [r] unacked
+    #   @return [Integer] messages in the plan's mailbox nobody has acked yet.
+    #     Data for the screen only (D2): no round, route or rename reads it,
+    #     because an unanswered message is a warning, never an input.
+    # @!attribute [r] running
+    #   @return [Array<String>] its sub-agents still at work, by name. Empty
+    #     once the row is done, and for an adapter whose stream never says
     Board::Row = Data.define(:key,
       :at,
       :ordinal,
@@ -75,12 +90,28 @@ module OpenLanes
       :history,
       :context,
       :since,
-      :session) do
+      :session,
+      :unacked,
+      :running) do
       def initialize(pr: nil, phase: nil, remaining: nil, frame: 0, bold: false, message: nil,
-        elapsed: 0, subagents: 0, history: [], context: nil, since: nil, session: nil, **rest)
+        elapsed: 0, subagents: 0, history: [], context: nil, since: nil, session: nil, unacked: 0, running: [], **rest)
         super
       end
 
+      # The count a row carries, read from the plan folder's mailbox, so the
+      # dispatcher fills the field in one call and the snapshot and the board
+      # cannot count differently (both ask Mailbox#unacked).
+      #
+      # @param dir [String] the plan folder
+      # @return [Integer] 0 for a plan nobody has written to, and for one
+      #   whose `state.json` cannot be read: a count for the screen must not
+      #   stop the tick that draws it
+      def self.unacked_in(dir)
+        Plans.mailbox(dir).unacked
+      rescue OpenLanes::Error, SystemCallError
+        0
+      end
+
       # @param history [Array<String>] newest first
       # @param text [String, nil]
       # @return [Array<String>] +text+ on top, unless it is empty or repeats

data/lib/openlanes/engine/conversions.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+
+module OpenLanes
+  module Engine
+    # The dispatcher's half of D9: an open question in a blocked plan's
+    # `blocked.md` that offers no choices is sent to the {Live::ChoiceConverter}
+    # and, once the model has answered, rewritten in place into D3's shape, so
+    # the page, lando-broker and everything else read the same options.
+    #
+    # The call runs on a thread of its own, like a drain, and its result is
+    # applied on a later tick, where the folder is looked up afresh. Nothing is
+    # written when the question changed meanwhile, was answered, or the plan
+    # was taken by another machine or another job.
+    #
+    # Each text is sent once. A rewrite, or an answer the model gave that is
+    # not a D3 question, is recorded in the plan's `state.json` under the
+    # SHA256 of the text, so a restarted run does not pay for it again. A call
+    # that failed (no network, a 4xx, a timeout) is remembered only for this
+    # process: the next run may well get through.
+    class Conversions
+      # One question sent to the model.
+      Job = Struct.new(:ordinal, :number, :digest, :original, :thread)
+
+      # @param converter [#call, nil] a {Live::ChoiceConverter}; nil converts nothing
+      # @param runner [OpenLanes::Engine::Runner]
+      # @param find [#call] an ordinal to the plan's {Plans::Subject} as the
+      #   folder is now, nil when it is gone
+      def initialize(converter:, runner:, find:)
+        @converter = converter
+        @runner = runner
+        @find = find
+        @jobs = []
+        @sent = []
+      end
+
+      # @return [Array<String>] plans with a question at the model right now
+      def ordinals = @jobs.map(&:ordinal)
+
+      # Apply what came back, then send what has not been sent.
+      #
+      # @param busy [Array<String>] plans nobody but their holder may touch
+      # @param start [Boolean] false applies answers and sends nothing new
+      # @return [void]
+      def call(busy:, start: true)
+        return if @converter.nil?
+
+        @jobs.reject { it.thread.alive? }.each { settle(it, busy) }
+        return unless start
+
+        @runner.blocked.each do |subject|
+          ordinal = subject.feature.ordinal.to_s
+          next if busy.include?(ordinal)
+
+          prose(subject).each do |question, body|
+            next unless @runner.claim(subject)
+
+            send_off(ordinal, question, body)
+          end
+        end
+      end
+
+      private
+
+      # Open questions with a body and no choices in it, not yet sent.
+      #
+      # @param subject [OpenLanes::Plans::Subject]
+      # @return [Array<Array(Lifecycle::Unblocker::Question, String)>]
+      def prose(subject)
+        bodies = Plans::BlockedFile.bodies(subject.read("blocked.md").to_s)
+        state = Plans::PlanState.for(subject.feature.path)
+        Lifecycle::Unblocker.questions(subject).reject(&:answered).filter_map do |question|
+          body = bodies.fetch(question.number, "").strip
+          next if body.empty? || question.heading.match?(Plans::Choices::YES_NO) || Plans::Choices.parse(body)
+
+          digest = Digest::SHA256.hexdigest(body)
+          next if @sent.include?(digest) || state.converted?(digest)
+
+          [question, body]
+        end
+      end
+
+      # @return [void]
+      def send_off(ordinal, question, body)
+        digest = Digest::SHA256.hexdigest(body)
+        @sent << digest
+        job = Job.new(ordinal:, number: question.number, digest:, original: body)
+        heading = question.heading
+        job.thread = Thread.new do
+          Thread.current.report_on_exception = false
+          @converter.call(heading:, body:)
+        rescue StandardError => e
+          e
+        end
+        @jobs << job
+      end
+
+      # @param job [Job]
+      # @param busy [Array<String>]
+      # @return [void]
+      def settle(job, busy)
+        @jobs.delete(job)
+        outcome = job.thread.value
+        subject = @find.call(Plans::Ordinal.parse(job.ordinal))
+        return couldnt(job, "the plan's folder is gone") if subject.nil?
+        return couldnt(job, "the plan was taken meanwhile") if busy.include?(job.ordinal)
+
+        case outcome
+        when Live::ChoiceConverter::Result then apply(job, subject, outcome)
+        when Live::ChoiceConverter::Invalid then reject(job, subject, outcome)
+        else couldnt(job, outcome.message.lines.first.to_s.strip)
+        end
+      end
+
+      # @return [void]
+      def apply(job, subject, result)
+        path = File.join(subject.feature.path, "blocked.md")
+        text = subject.read("blocked.md").to_s
+        open = Lifecycle::Unblocker.questions(subject).find { it.number == job.number && !it.answered }
+        current = Plans::BlockedFile.bodies(text).fetch(job.number, "").strip
+        return couldnt(job, "it changed or was answered while the model was writing") if !open || current != job.original
+
+        write(path, Plans::BlockedFile.replace(text, job.number, result.body))
+        Plans::PlanState.for(subject.feature.path).conversion!(question: job.number,
+          digest: job.digest,
+          original: job.original,
+          model: result.model,
+          outcome: "converted")
+        UI.log("questions: converted B#{job.number} to #{result.count} choices", plan: job.ordinal)
+      end
+
+      # @return [void]
+      def reject(job, subject, error)
+        Plans::PlanState.for(subject.feature.path).conversion!(question: job.number,
+          digest: job.digest,
+          original: job.original,
+          model: @converter.model,
+          outcome: "rejected",
+          note: error.message)
+        couldnt(job, error.message)
+      end
+
+      # @return [void]
+      def couldnt(job, why) = UI.log("questions: could not convert B#{job.number}: #{why}", level: :warn, plan: job.ordinal)
+
+      # A reader never sees half a file.
+      #
+      # @return [void]
+      def write(path, text)
+        tmp = "#{path}.#{Process.pid}.tmp"
+        File.write(tmp, text)
+        File.rename(tmp, path)
+      ensure
+        FileUtils.rm_f(tmp) if tmp && File.exist?(tmp)
+      end
+    end
+  end
+end