openlanes 3.0.0

data/lib/openlanes/live/run_record.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require "json"
+
+module OpenLanes
+  module Live
+    # Keeps the {RunState} past an engine restart (W6). Redis holds the live
+    # copy under {Keys#run}; `.plans/openlanes-state.json` holds the one a
+    # person can read without `redis-cli` and the one that survives Redis
+    # being flushed. Both carry the state's `seq`.
+    #
+    # On the run's first tick, after the dispatcher has loaded its state
+    # file, the newer copy is taken: normally Redis, the file when Redis came
+    # back empty or behind. Queued models go back to the executor, which is
+    # what applies them. From then on each tick writes whichever copy is
+    # behind, and {Commands} writes Redis in the same `MULTI` as a command's
+    # reply, so an applied command and its effect on the run state land
+    # together.
+    class RunRecord
+      # @param store [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore]
+      # @param keys [OpenLanes::Live::Keys]
+      # @param run_state [OpenLanes::Live::RunState]
+      def initialize(store:, keys:, run_state:)
+        @store = store
+        @keys = keys
+        @run_state = run_state
+        @restored = false
+        @in_redis = nil
+        @in_file = nil
+        @state_file = nil
+        @warned = false
+      end
+
+      # @return [Proc] for {Engine::Dispatcher#before_tick}
+      def to_proc = method(:call).to_proc
+
+      # @param dispatcher [OpenLanes::Engine::Dispatcher]
+      # @return [void]
+      def call(dispatcher)
+        restore(dispatcher) unless @restored
+        flush
+      end
+
+      # Take back the newer copy, and hand its models to the executor.
+      #
+      # @param dispatcher [OpenLanes::Engine::Dispatcher]
+      # @return [void]
+      def restore(dispatcher)
+        @restored = true
+        @state_file = dispatcher.state_file
+        redis = read_redis
+        file = @state_file&.live
+        newer = [redis, file].compact.max_by { it["seq"].to_i }
+        # A copy that is missing is behind, unless both are: then there is
+        # nothing to keep until a person changes something.
+        @in_redis = redis ? redis["seq"].to_i : (newer && -1) || 0
+        @in_file = file ? file["seq"].to_i : (newer && -1) || 0
+        return unless newer
+
+        @run_state.restore(newer)
+        @run_state.models.each do |key, model|
+          ordinal, agent = key.split("/", 2)
+          dispatcher.prefer_model(ordinal, agent, model)
+        end
+      end
+
+      # Bring both copies up to the state's `seq`. The file is written by the
+      # dispatcher's own save at the end of the tick, and by {#close}.
+      #
+      # @return [void]
+      def flush
+        seq = @run_state.seq
+        @store.multi { |tx| write(tx) } unless @in_redis == seq
+        mirror
+      rescue Unavailable => e
+        unavailable(e)
+      end
+
+      # The Redis half, inside someone else's `MULTI`.
+      #
+      # @param transaction [#hset]
+      # @return [void]
+      def write(transaction)
+        data = @run_state.to_h
+        transaction.hset(@keys.run, "seq", data["seq"].to_s)
+        transaction.hset(@keys.run, "json", JSON.generate(data))
+        @in_redis = data["seq"]
+      end
+
+      # The run is over: the file gets the last word, saved now rather than
+      # at a tick that is not coming.
+      #
+      # @return [void]
+      def close
+        flush
+        @state_file&.save
+      end
+
+      private
+
+      # @return [void]
+      def mirror
+        return unless @state_file && @in_file != @run_state.seq
+
+        data = @run_state.to_h
+        @state_file.live = data
+        @in_file = data["seq"]
+      end
+
+      # @return [Hash, nil]
+      def read_redis
+        seq, json = @store.hgetall(@keys.run).values_at("seq", "json")
+        return nil unless json
+
+        JSON.parse(json).merge("seq" => seq.to_i)
+      rescue JSON::ParserError
+        nil
+      rescue Unavailable => e
+        unavailable(e)
+        nil
+      end
+
+      # @param error [OpenLanes::Live::Unavailable]
+      # @return [void]
+      def unavailable(error)
+        return if @warned
+
+        @warned = true
+        UI.log("run state not kept in Redis: #{error.message}")
+      end
+    end
+  end
+end

data/lib/openlanes/live/run_state.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Live
+    # What a person has said about the run that the plan folders do not
+    # record: the questions they are holding, the agents they asked to hold,
+    # and the models chosen for agents' next rounds.
+    #
+    # Held in the engine, and kept by {RunRecord} in Redis and in
+    # `.plans/openlanes-state.json` (W6), so an engine that restarts still
+    # knows them. Every change moves {#seq} on by one, which is how the
+    # record tells which of its two copies is newer. A model choice also
+    # lives in the executor, which is what applies it; this copy is what the
+    # page is shown and what a restart hands back to the executor.
+    class RunState
+      def initialize
+        @mutex = Mutex.new
+        @held_questions = []
+        @held_cars = []
+        @models = {}
+        @seq = 0
+      end
+
+      # @param id [String] `"<ordinal>/B<n>"`
+      # @param held [Boolean]
+      # @return [void]
+      def hold_question(id, held: true) = toggle(@held_questions, id, held)
+
+      # @param id [String]
+      # @return [Boolean]
+      def question_held?(id) = @mutex.synchronize { @held_questions.include?(id) }
+
+      # @param key [String] `"<ordinal>/<agent>"`
+      # @param held [Boolean]
+      # @return [void]
+      def hold_car(key, held: true) = toggle(@held_cars, key, held)
+
+      # @param key [String]
+      # @return [Boolean]
+      def car_held?(key) = @mutex.synchronize { @held_cars.include?(key) }
+
+      # @param key [String] `"<ordinal>/<agent>"`
+      # @param model [String, nil] nil forgets the choice
+      # @return [void]
+      def choose_model(key, model)
+        @mutex.synchronize do
+          next if @models[key] == model
+
+          model ? @models[key] = model : @models.delete(key)
+          @seq += 1
+        end
+      end
+
+      # @param key [String]
+      # @return [String, nil] the model queued for that agent's next round
+      def model_for(key) = @mutex.synchronize { @models[key] }
+
+      # @return [Hash{String => String}] `"<ordinal>/<agent>"` to model, a copy
+      def models = @mutex.synchronize { @models.dup }
+
+      # @return [Integer] changes made since the run's state was first recorded
+      def seq = @mutex.synchronize { @seq }
+
+      # As {RunRecord} keeps it.
+      #
+      # @return [Hash]
+      def to_h
+        @mutex.synchronize do
+          { "seq" => @seq, "held_questions" => @held_questions.dup, "held_cars" => @held_cars.dup,
+            "models" => @models.dup }
+        end
+      end
+
+      # Take back what {#to_h} wrote, sequence number included.
+      #
+      # @param data [Hash]
+      # @return [self]
+      def restore(data)
+        @mutex.synchronize do
+          @held_questions = Array(data["held_questions"]).map(&:to_s)
+          @held_cars = Array(data["held_cars"]).map(&:to_s)
+          @models = (data["models"].is_a?(Hash) ? data["models"] : {}).to_h { |k, v| [k.to_s, v.to_s] }
+          @seq = data["seq"].to_i
+        end
+        self
+      end
+
+      private
+
+      # @return [void]
+      def toggle(list, item, on)
+        @mutex.synchronize do
+          next if list.include?(item) == on
+
+          on ? list.push(item) : list.delete(item)
+          @seq += 1
+        end
+      end
+    end
+  end
+end

data/lib/openlanes/live/session.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Live
+    # One engine's live run: its node and keys, its claims, the publisher
+    # that writes its board and the reader that applies its commands, built
+    # once and handed to the runner.
+    #
+    #   live = Live.session(store:, root:, tree:, agents:)
+    #   runner = Engine.runner(..., on_board: live.on_board, claims: live.claims)
+    #   runner.call { |dispatcher| live.attach(dispatcher) }
+    #   live.close
+    class Session
+      # @param store [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore]
+      # @param root [String] the checkout
+      # @param tree [OpenLanes::Plans::Tree]
+      # @param agents [OpenLanes::Agents::Registry]
+      # @param project [String, nil] the header's name (W18); the root's name by default
+      # @param node [OpenLanes::Live::Node]
+      # @param repo [String]
+      # @param clock [Proc] wall time
+      def initialize(store:, root:, tree:, agents:, project: nil, node: Node.current(root:), repo: Repo.for(root),
+        clock: -> { Time.now })
+        @store = store
+        @node = node
+        @keys = Keys.new(repo:, node: node.name)
+        @run_state = RunState.new
+        @record = RunRecord.new(store:, keys: @keys, run_state: @run_state)
+        @claims = Claims.new(store:, keys: @keys)
+        @publisher = Publisher.new(store:,
+          keys: @keys,
+          node:,
+          tree:,
+          agents:,
+          claims: @claims,
+          run_state: @run_state,
+          project: project || File.basename(root),
+          clock:)
+        @commands = Commands.new(store:,
+          keys: @keys,
+          node:,
+          run_state: @run_state,
+          record: @record,
+          claims: @claims,
+          clock:)
+      end
+
+      # @return [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore]
+      attr_reader :store
+
+      # @return [OpenLanes::Live::Node]
+      attr_reader :node
+
+      # @return [OpenLanes::Live::Keys]
+      attr_reader :keys
+
+      # @return [OpenLanes::Live::Claims]
+      attr_reader :claims
+
+      # @return [OpenLanes::Live::Publisher]
+      attr_reader :publisher
+
+      # @return [OpenLanes::Live::Commands]
+      attr_reader :commands
+
+      # @return [OpenLanes::Live::RunState]
+      attr_reader :run_state
+
+      # @return [OpenLanes::Live::RunRecord]
+      attr_reader :record
+
+      # @return [Proc] the runner's `on_board:`
+      def on_board = @publisher.to_proc
+
+      # Wire the run in: the run state a previous engine left is taken back
+      # on the first tick, commands are applied at the top of each tick, and
+      # the log is heard from now on.
+      #
+      # @param dispatcher [OpenLanes::Engine::Dispatcher]
+      # @return [void]
+      def attach(dispatcher)
+        @publisher.listen
+        @publisher.on_lost { |ordinals| dispatcher.fence(ordinals) }
+        dispatcher.before_tick(&@record)
+        dispatcher.before_tick(&@commands)
+      end
+
+      # The run is over: the run state is written down, claims go back, and
+      # this node's snapshot goes at once rather than when it expires, so
+      # other pages stop showing it.
+      #
+      # @return [void]
+      def close
+        @record.close
+        @publisher.close
+        @claims.release_all
+        @store.del(@keys.snapshot)
+        @store.hdel(@keys.nodes, @node.name)
+      rescue Unavailable
+        nil
+      end
+
+      # @return [String] where, with the password masked, and as whom
+      def describe = "#{Live.mask(@store.url)} as #{@node.label} (#{@node.name})"
+    end
+  end
+end

data/lib/openlanes/live/snapshot.rb

@@ -0,0 +1,322 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Live
+    # Everything the page shows about this node, as one hash in exactly the
+    # shape of `Snapshot` in `web/src/api/types.ts`.
+    #
+    # Built from what already says it: the dispatcher's {Engine::Board} for
+    # who is running and what they spent, the `.plans` tree for each plan's
+    # state, questions and pull requests, the registry for the stations, the
+    # adapter for the models and their prices. Nothing here is a second copy
+    # of any of them. `spec/openlanes/live/snapshot_spec.rb` holds the keys
+    # to the page's fixture, level by level, so a field renamed on one side
+    # fails the other's suite.
+    class Snapshot
+      # How an agent marks a question as answerable with Yes or No (W30), on
+      # its own line under the question's heading. The agents' prompt shows
+      # it, and {#questions} looks for it; neither spells it twice.
+      YES_NO_MARK = "answers: yes, no"
+
+      # {YES_NO_MARK}, read forgivingly: brackets, case and spacing vary.
+      YES_NO = /answers:\s*\[?\s*yes\s*,\s*no\s*\]?/i
+
+      # Any Markdown heading, which ends the body of the question above it.
+      HEADING = /^[ \t]{0,3}\#{1,6}[ \t]/
+
+      # Plan states finished with, kept off "Choose an existing plan".
+      FINISHED = %i[approved deployed discarded].freeze
+
+      # @param board [OpenLanes::Engine::Board]
+      # @param tree [OpenLanes::Plans::Tree]
+      # @param agents [OpenLanes::Agents::Registry]
+      # @param node [OpenLanes::Live::Node]
+      # @param repo [String]
+      # @param project [String]
+      # @param description [String, nil]
+      # @param seq [Integer]
+      # @param at [Time]
+      # @param started_at [Time, nil] the run's start, wall clock
+      # @param claims [OpenLanes::Live::Claims, nil]
+      # @param run_state [OpenLanes::Live::RunState]
+      # @param meter [OpenLanes::Live::Meter]
+      # @param log [Array<Hash>] `LogLine`s, newest last
+      # @param adapter [OpenLanes::Adapters::Base] whose models the page offers
+      def initialize(board:, tree:, agents:, node:, repo:, project:, seq:, description: nil, at: Time.now,
+        started_at: nil, claims: nil, run_state: RunState.new, meter: Meter.new, log: [],
+        adapter: Adapters.for(Adapters::DEFAULT))
+        @board = board
+        @tree = tree
+        @agents = agents
+        @node = node
+        @repo = repo
+        @project = project
+        @description = description
+        @seq = seq
+        @at = at
+        @started_at = started_at
+        @claims = claims
+        @run_state = run_state
+        @meter = meter
+        @log = log
+        @adapter = adapter
+      end
+
+      # @return [Hash]
+      def to_h
+        subjects = @tree.reload.subjects
+        mine = ours(subjects)
+        { seq:         @seq,
+          at:          iso(@at),
+          node:        @node.wire(heartbeat_at: @at, me: true),
+          repo:        @repo,
+          project:     @project,
+          description: @description,
+          run:         run(mine),
+          stations:    Stations.all(@agents).map(&:wire),
+          plans:       mine,
+          backlog:     backlog(subjects, mine.map { it[:ordinal] }),
+          models:      @adapter.models.map(&:wire),
+          autocompact: Adapters::AUTOCOMPACT / 100.0,
+          log:         @log }
+      end
+
+      private
+
+      # ---- run --------------------------------------------------------------
+
+      # @param plans [Array<Hash>]
+      # @return [Hash]
+      def run(plans)
+        { status:         run_status,
+          started_at:     @started_at && iso(@started_at),
+          up:             @board.up.to_i,
+          down:           @board.down.to_i,
+          cost:           plans.sum { it[:cost] }.round(4),
+          agents_working: @board.running.to_i }
+      end
+
+      # @return [String]
+      def run_status
+        return "idle" if @board.running.to_i.zero? && @board.rows.empty? && @board.status == :running
+
+        @board.status.to_s
+      end
+
+      # ---- plans ------------------------------------------------------------
+
+      # The plans this node is working on: the board's, and any it has
+      # claimed but not yet started.
+      #
+      # @param subjects [Array<OpenLanes::Plans::Subject>]
+      # @return [Array<Hash>]
+      def ours(subjects)
+        ordinals = (@board.plans + Array(@claims&.held)).uniq
+        subjects.select { |s| ordinals.include?(s.feature.ordinal.to_s) }.map { |s| plan(s) }
+      end
+
+      # @param subject [OpenLanes::Plans::Subject]
+      # @return [Hash] `Plan`
+      def plan(subject)
+        ordinal = subject.feature.ordinal.to_s
+        state = Plans::PlanState.for(subject.feature.path)
+        stages = state.stages
+        rows = @board.rows.select { it.ordinal == ordinal }
+        spent = @meter.for(ordinal, method(:price))
+        started = @meter.started_at(ordinal)
+        { ordinal:,
+          title:       subject.feature.title,
+          slug:        subject.feature.slug,
+          status:      status(subject.status),
+          lane:        Plans::Spec.for(subject).lane.to_s,
+          station:     Stations.key_for(subject.status.key, before: stages.last&.fetch("from", nil)&.to_sym).to_s,
+          up:          spent[:up],
+          down:        spent[:down],
+          cost:        spent[:cost],
+          started_at:  started && iso(started),
+          elapsed:     started ? (@at - started).round : 0,
+          cars:        rows.map { car(it, subject, stages) },
+          pr:          pull_request(subject),
+          questions:   questions(subject, state),
+          interrupted: interrupted(stages, rows),
+          claimed_by:  claimed_by(ordinal) }
+      end
+
+      # @param status [OpenLanes::Plans::Status]
+      # @return [Hash] `Status`
+      def status(status) = { key: status.key.to_s, emoji: status.emoji, label: status.label }
+
+      # @param subjects [Array<OpenLanes::Plans::Subject>]
+      # @param taken [Array<String>] ordinals already on the board
+      # @return [Array<Hash>] plans a person could add to this run
+      def backlog(subjects, taken)
+        subjects.reject { |s| taken.include?(s.feature.ordinal.to_s) || FINISHED.include?(s.status.key) }
+                .reject { |s| foreign?(s.feature.ordinal.to_s) }
+                .map do |s|
+          { ordinal: s.feature.ordinal.to_s, title: s.feature.title, status: status(s.status),
+            lane: Plans::Spec.for(s).lane.to_s }
+        end
+      end
+
+      # ---- cars -------------------------------------------------------------
+
+      # @param row [OpenLanes::Engine::Board::Row]
+      # @param subject [OpenLanes::Plans::Subject]
+      # @param stages [Array<Hash>]
+      # @return [Hash] `Car`
+      def car(row, subject, stages)
+        { key:           row.key,
+          agent:         row.agent,
+          role:          row.role.to_s,
+          round:         row.round,
+          rounds:        row.rounds,
+          model:         model_id(row.model),
+          effort:        stages.rfind { it["agent"] == row.agent }&.fetch("effort", nil),
+          state:         car_state(row, subject),
+          phase:         row.phase&.to_s,
+          message:       row.message.to_s,
+          history:       Array(row.history).first(Engine::Board::Row::HISTORY),
+          up:            row.up.to_i,
+          down:          row.down.to_i,
+          context:       row.context,
+          context_limit: price(row.model)&.context,
+          elapsed:       row.elapsed.to_i,
+          remaining:     row.remaining,
+          held:          @run_state.car_held?(row.key),
+          next_model:    next_model(row) }
+      end
+
+      # The model a person queued for the agent's next round, until a round
+      # runs on it: from then on it is the model, not the next one.
+      #
+      # @param row [OpenLanes::Engine::Board::Row]
+      # @return [String, nil]
+      def next_model(row)
+        queued = @run_state.model_for(row.key)
+        queued unless queued == model_id(row.model)
+      end
+
+      # A running agent on a plan that has stopped for a question is stuck,
+      # not working: the page says so rather than spinning.
+      #
+      # @return [String]
+      def car_state(row, subject)
+        return "stuck" if row.running? && %i[blocked product_blocked].include?(subject.status.key)
+
+        row.state.to_s
+      end
+
+      # @param label [String, nil] a {Agents::Profile#label}: `opus`, or `codex:gpt-5`
+      # @return [String] the full model id when the adapter knows it
+      def model_id(label)
+        name = label.to_s.split(":", 2).last
+        adapter_for(label)&.model_entry(name)&.id || name
+      end
+
+      # The catalogue's entry for a model: its prices, and its context window.
+      #
+      # @param label [String, nil]
+      # @return [OpenLanes::Adapters::Model, nil]
+      def price(label) = adapter_for(label)&.model_entry(label.to_s.split(":", 2).last)
+
+      # @param label [String, nil]
+      # @return [OpenLanes::Adapters::Base, nil]
+      def adapter_for(label)
+        prefix = label.to_s.include?(":") ? label.to_s.split(":", 2).first : Adapters::DEFAULT
+        Adapters.known?(prefix) ? Adapters.for(prefix) : nil
+      end
+
+      # ---- pull request, interruption, claim --------------------------------
+
+      # @param subject [OpenLanes::Plans::Subject]
+      # @return [Hash, nil] `PullRequest`
+      def pull_request(subject)
+        pr = subject.pull_requests.find(&:open?) || subject.pull_requests.last or return nil
+        { number: pr.number.to_s[/\d+/].to_i, url: pr.url, state: pr_state(pr, subject), ci: nil, merged_at: nil }
+      end
+
+      # @return [String]
+      def pr_state(pr, subject)
+        return "merged" if pr.merged?
+        return "closed" unless pr.open?
+
+        { rejected: "rejected", approved: "approved" }.fetch(subject.status.key, "open")
+      end
+
+      # The latest stage, when it is an interruption nobody has resumed.
+      #
+      # @param stages [Array<Hash>]
+      # @param rows [Array<OpenLanes::Engine::Board::Row>]
+      # @return [Hash, nil]
+      def interrupted(stages, rows)
+        last = stages.last
+        return nil unless last && last["status"] == "Interrupted" && rows.none? { it.agent == last["agent"] && it.running? }
+
+        { agent: last["agent"], at: last["finished_at"] || last["started_at"], note: last["note"].to_s }
+      end
+
+      # @param ordinal [String]
+      # @return [String, nil] the node id, without its checkout
+      def claimed_by(ordinal) = @claims&.held_by(ordinal)&.then { it.split(":").first }
+
+      # @param ordinal [String]
+      # @return [Boolean]
+      def foreign?(ordinal) = !@claims.nil? && @claims.foreign?(ordinal)
+
+      # ---- questions --------------------------------------------------------
+
+      # The questions `blocked.md` still asks, each with its body, whether it
+      # is a yes/no question, and who asked.
+      #
+      # @param subject [OpenLanes::Plans::Subject]
+      # @param state [OpenLanes::Plans::PlanState]
+      # @return [Array<Hash>] `Question`
+      def questions(subject, state)
+        open = Lifecycle::Unblocker.questions(subject).reject(&:answered)
+        return [] if open.empty?
+
+        bodies = bodies(subject.read("blocked.md").to_s)
+        asker = state.signatures.rfind { it["status"] == "Blocked" }
+        asked_at = asker&.fetch("at", nil) || iso(File.mtime(File.join(subject.feature.path, "blocked.md")))
+        open.map { |q| question(subject, q, bodies.fetch(q.number, ""), asker, asked_at) }
+      end
+
+      # @return [Hash]
+      def question(subject, open, body, asker, asked_at)
+        id = "#{subject.feature.ordinal}/B#{open.number}"
+        yes_no = open.heading.match?(YES_NO) || body.match?(YES_NO)
+        { id:,
+          number:   open.number,
+          heading:  open.heading.sub(YES_NO, "").strip,
+          body:     body.lines.grep_v(/\A\s*#{YES_NO.source}\s*\z/i).join.strip,
+          agent:    asker&.fetch("agent", nil),
+          owner:    subject.status.key == :product_blocked ? "product" : "technical",
+          answers:  yes_no ? %w[yes no] : nil,
+          held:     @run_state.question_held?(id),
+          asked_at: }
+      end
+
+      # @param text [String] the whole of blocked.md
+      # @return [Hash{Integer => String}] each open question's number to the
+      #   lines under its heading, up to the next heading
+      def bodies(text)
+        found = {}
+        current = nil
+        text.each_line do |line|
+          if (number = line[Plans::OPEN_BLOCK, 1])
+            current = found[number.to_i] = +""
+          elsif line.match?(HEADING) || line.match?(Plans::ANSWER_BLOCK)
+            current = nil
+          elsif current
+            current << line
+          end
+        end
+        found
+      end
+
+      # @param time [Time]
+      # @return [String]
+      def iso(time) = time.utc.iso8601
+    end
+  end
+end