openlanes 3.0.0 → 3.1.0

data/lib/openlanes/engine/event_journal/replay.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require "json"
+
+module OpenLanes
+  module Engine
+    class EventJournal
+      # Rebuilds the in-flight picture from a journal file: the run, each
+      # plan's last known state, and every agent that started and has not
+      # ended, with its pid, session and last offset.
+      #
+      # A kill can tear the last line, so a line that does not parse is
+      # counted in `torn` and skipped. A line whose `v` is newer than
+      # {EventJournal::VERSION} is counted in `newer` rather than read, so an
+      # old reader never invents a picture from fields it does not know.
+      module Replay
+        # One agent the journal says is running.
+        Agent = Data.define(:plan, :agent, :round, :pid, :started_at, :session_id, :output_file, :offset)
+
+        # What the journal says was going on when it stopped.
+        #
+        # @!attribute [r] run
+        #   @return [Hash, nil] the `run_started` event, and `ended` when a `run_ended` followed
+        # @!attribute [r] plans
+        #   @return [Hash{String => String}] plan number to the last state a transition named
+        # @!attribute [r] agents
+        #   @return [Array<Agent>] agents started and not ended, in start order
+        # @!attribute [r] torn
+        #   @return [Integer] lines that did not parse
+        # @!attribute [r] newer
+        #   @return [Integer] lines from a newer schema version, skipped
+        # @!attribute [r] seq
+        #   @return [Integer] the last `seq` read
+        InFlight = Data.define(:run, :plans, :agents, :torn, :newer, :seq) do
+          # @return [Boolean] the run started and no `run_ended` followed
+          def running? = !run.nil? && !run.key?("ended")
+        end
+
+        module_function
+
+        # @param path [String] a journal file
+        # @return [InFlight]
+        def call(path)
+          state = { run: nil, plans: {}, agents: {}, torn: 0, newer: 0, seq: 0 }
+          File.foreach(path) { read(state, it) }
+          InFlight.new(run: state[:run],
+            plans: state[:plans],
+            agents: state[:agents].values,
+            torn: state[:torn],
+            newer: state[:newer],
+            seq: state[:seq])
+        end
+
+        # @api private
+        def read(state, line)
+          return if line.strip.empty?
+
+          event = JSON.parse(line)
+          return state[:newer] += 1 if event["v"].to_i > VERSION
+
+          state[:seq] = event["seq"].to_i
+          apply(state, event)
+        rescue JSON::ParserError
+          state[:torn] += 1
+        end
+
+        # @api private
+        def apply(state, event)
+          key = [event["plan"], event["agent"]]
+          agents = state[:agents]
+          case event["kind"]
+          when "run_started" then state[:run] = event
+          when "run_ended" then state[:run] = (state[:run] || {}).merge("ended" => event)
+          when "transition" then state[:plans][event["plan"]] = event["to"]
+          when "agent_started" then agents[key] = start(event)
+          when "agent_session" then agents[key] &&= agents[key].with(session_id: event["session_id"])
+          when "agent_offset" then agents[key] &&= agents[key].with(offset: event["offset"])
+          when "agent_ended" then agents.delete(key)
+          end
+        end
+
+        # @api private
+        def start(event)
+          Agent.new(plan: event["plan"],
+            agent: event["agent"],
+            round: event["round"],
+            pid: event["pid"],
+            started_at: event["started_at"],
+            session_id: event["session_id"],
+            output_file: event["output_file"],
+            offset: 0)
+        end
+      end
+    end
+  end
+end

data/lib/openlanes/engine/event_journal.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "securerandom"
+require "socket"
+require "time"
+
+module OpenLanes
+  module Engine
+    # The node's progress journal: one append-only NDJSON file per run under
+    # {Runtime.journal_dir}, one schema-versioned line per engine event,
+    # flushed and fsync'd before {#append} returns. Each line is then mirrored
+    # to the {Live::EventBus} on {Live::Keys#journal}.
+    #
+    # The file is the record, not Redis. {StateFile} is a snapshot rewritten
+    # every tick, and the Redis board keeps a hundred entries that vanish with
+    # the server, so neither could say what a killed engine had been doing.
+    # The bus is best-effort: a publish that raises is reported once and the
+    # line stays on disk.
+    #
+    # Named `EventJournal`, never `Journal`: {Plans::Journal} is the agents'
+    # own memory in `state.json`, a different thing that shares the word.
+    #
+    # @example
+    #   journal = EventJournal.new(root: Dir.pwd, run_id: "r1", node: "mbp:abc123")
+    #   journal.append(:transition, plan: "003.00", from: "planned", to: "building", event: "build")
+    #   EventJournal::Replay.call(journal.path).agents
+    class EventJournal
+      # The schema every line validates against.
+      SCHEMA_PATH = File.expand_path("../../../schemas/journal-event.schema.json", __dir__)
+
+      # The `v` of every line this writer appends. A reader that meets a
+      # newer one reports it rather than guessing at the fields.
+      VERSION = 1
+
+      # Every `kind` the schema accepts, in the order a run produces them.
+      KINDS = %w[
+        run_started run_ended plan_assigned agent_started agent_session
+        agent_offset agent_ended transition extension question
+      ].freeze
+
+      # @return [String] the NDJSON file this journal appends to
+      attr_reader :path
+
+      # @return [String]
+      attr_reader :run_id
+
+      # @param root [String] the project root, keying {Runtime.journal_dir}
+      # @param run_id [String] names the file; one file per run, never truncated
+      # @param node [String] this node's `<node id>:<checkout id>`
+      # @param bus [Live::EventBus::Base, nil] where each line is mirrored
+      # @param channel [String, nil] the bus channel, {Live::Keys#journal}
+      # @param clock [#now]
+      def initialize(root:, run_id: self.class.run_id, node: self.class.node, bus: nil, channel: nil, clock: Time)
+        @run_id = run_id
+        @node = node
+        @bus = bus
+        @channel = channel
+        @clock = clock
+        @path = File.join(Runtime.journal_dir(root:), "#{run_id}.ndjson")
+        @lock = Mutex.new
+        @seq = last_seq
+        @warned = false
+        # O_APPEND, so each line lands whole at the end even beside another
+        # writer, and one `write` per line, so a kill tears at most the last.
+        @file = File.open(@path, File::WRONLY | File::APPEND | File::CREAT, 0o644)
+        seal_torn_tail
+      end
+
+      # Append one event, durable on return.
+      #
+      # @param kind [String, Symbol] one of {KINDS}
+      # @param fields [Hash] the kind's own fields; `nil` values are kept,
+      #   because the schema tells "unknown" from "absent"
+      # @return [Hash] the event as written
+      # @raise [ArgumentError] on a kind outside {KINDS}
+      def append(kind, **fields)
+        kind = kind.to_s
+        raise ArgumentError, "unknown journal event kind: #{kind}" unless KINDS.include?(kind)
+
+        event = @lock.synchronize do
+          @seq += 1
+          line = { "v" => VERSION, "seq" => @seq, "at" => @clock.now.utc.iso8601(6), "node" => @node,
+                   "run_id" => @run_id, "kind" => kind }.merge(fields.transform_keys(&:to_s))
+          @file.write("#{JSON.generate(line)}\n")
+          @file.flush
+          @file.fsync
+          line
+        end
+        mirror(event)
+        event
+      end
+
+      # @return [void]
+      def close = @lock.synchronize { @file.close unless @file.closed? }
+
+      # @param root [String]
+      # @return [String, nil] the most recently written journal under +root+'s runtime directory
+      def self.latest(root:)
+        Dir.glob(File.join(Runtime.journal_dir(root:), "*.ndjson")).max_by { File.mtime(it) }
+      end
+
+      # @param clock [#now]
+      # @return [String] `YYYYMMDD-HHMMSS-<6 hex>`, sortable and unique per run
+      def self.run_id(clock: Time) = "#{clock.now.utc.strftime("%Y%m%d-%H%M%S")}-#{SecureRandom.hex(3)}"
+
+      # @return [String] the host and this process, when no live node id is known
+      def self.node = "#{Socket.gethostname}:#{Process.pid}"
+
+      private
+
+      # A file reopened for the same run carries on numbering where it stopped.
+      def last_seq
+        return 0 unless File.exist?(@path)
+
+        File.foreach(@path).reduce(0) do |seq, line|
+          JSON.parse(line)["seq"].to_i
+        rescue JSON::ParserError
+          seq
+        end
+      end
+
+      # A run killed mid-write leaves a fragment with no newline. The next
+      # line appended straight after it fused with it, and the replay then
+      # dropped both as one torn line, losing an event it had acknowledged.
+      def seal_torn_tail
+        return if File.empty?(@path)
+        return if File.open(@path, "rb") { it.seek(-1, IO::SEEK_END) && it.read(1) } == "\n"
+
+        @file.write("\n")
+        @file.flush
+        @file.fsync
+      end
+
+      def mirror(event)
+        return unless @bus && @channel
+
+        @bus.publish(@channel, JSON.generate(event))
+      rescue StandardError => e
+        return if @warned
+
+        @warned = true
+        UI.log("journal mirror to the event bus failed (#{e.class}: #{e.message}); the file still has every event")
+      end
+    end
+  end
+end

data/lib/openlanes/engine/extensions.rb

@@ -0,0 +1,271 @@
+# frozen_string_literal: true
+
+require "time"
+
+module OpenLanes
+  module Engine
+    # An agent that runs out of time or rounds asks for more instead of
+    # simply ending (E1). The harness, not the agent, asks: a question on the
+    # plan, `X<n>`, with three fixed choices, kept in the plan's `state.json`
+    # so it survives a restart and can be answered from any control room.
+    #
+    # - Time: asked while the agent still runs, {ASK_AT} seconds before its
+    #   clock's zero. A time choice answered then moves the live deadline, and
+    #   the agent carries on in the same session.
+    # - Rounds: asked once the agent has used every round it had on the plan's
+    #   state without finishing. Every choice gives it one more round; the
+    #   time choices also lengthen that round's clock.
+    #
+    # Nobody may be watching, so each kind is granted once without an answer:
+    # ten more minutes when the clock reaches {AUTO_AT}, and one more round at
+    # once on a run with no control room, or {AUTO_AFTER} seconds after asking
+    # on one that has one. After that the request waits for a person, and the
+    # limit applies as it always did: STOP at zero, no round past the last.
+    #
+    # Nothing here starts, stops or renames anything; the dispatcher asks and
+    # acts on the answers.
+    class Extensions
+      # Seconds left on a clock when the time request is asked: the same
+      # moment the agent itself is told "5 minutes left".
+      ASK_AT = 300
+
+      # Seconds left when an unanswered time request is granted on its own:
+      # before the clock's WRAP_UP at 60, so the agent is not already
+      # wrapping up when the time arrives.
+      AUTO_AT = 120
+
+      # Seconds a rounds request waits for a person on a staying run before
+      # it is granted on its own.
+      AUTO_AFTER = 300
+
+      # Who answered a request nobody else did.
+      AUTO = "auto"
+
+      # One of the three choices every request offers.
+      #
+      # @!attribute [r] letter
+      #   @return [String]
+      # @!attribute [r] label
+      #   @return [String]
+      # @!attribute [r] seconds
+      #   @return [Integer] extra seconds on the clock, 0 for a round alone
+      Choice = Data.define(:letter, :label, :seconds)
+
+      # @return [Array<Choice>]
+      CHOICES = [
+        Choice.new(letter: "a", label: "Extend Rounds by 1", seconds: 0),
+        Choice.new(letter: "b", label: "Extend Time by 10 minutes", seconds: 600),
+        Choice.new(letter: "c", label: "Extend Time by 20 minutes", seconds: 1200)
+      ].freeze
+
+      # The choice granted without an answer, by what ran out.
+      DEFAULTS = { "time" => "b", "rounds" => "a" }.freeze
+
+      # @param stay [Boolean] whether a control room can answer at all
+      # @param auto [Boolean] false to ask and never grant on its own
+      # @param now [Proc] the wall clock
+      def initialize(stay:, auto: true, now: -> { Clock::System.now })
+        @stay = stay
+        @auto = auto
+        @now = now
+        # Rounds used are counted in memory, from this run's start; rounds
+        # granted are counted from the same moment, or a restart would hand
+        # back every round ever granted on top of a fresh allowance.
+        @since = now.call
+      end
+
+      # A running agent whose clock is nearly out: ask once per round, and
+      # grant ten minutes on its own once per agent and state.
+      #
+      # @param job [#task, #state, #handle] the dispatcher's job
+      # @param path [String] the plan folder as it is named now
+      # @return [Hash, nil] the request, when there is one
+      def watch(job, path)
+        left = job.handle.remaining or return nil
+        return nil if left > ASK_AT
+
+        plan = Plans::PlanState.for(path)
+        agent = job.task.agent.name
+        state = job.state.to_s
+        request = pending(plan.extensions, agent, state) ||
+                  unless asked?(plan.extensions, agent, state, job.task.round, "time")
+                    plan.ask_extension!(agent:, state:, round: job.task.round, limit: "time", at: @now.call)
+                  end
+        return request unless @auto && request && left <= AUTO_AT && !auto?(plan.extensions, agent, state, "live")
+
+        answer!(path, request["number"], DEFAULTS.fetch("time"), by: AUTO, live: job.handle)
+      end
+
+      # An agent with no rounds left on the plan's state: ask once per
+      # exhaustion, and grant a round on its own once per agent and state.
+      #
+      # @param path [String]
+      # @param agent [String]
+      # @param state [#to_s] the status key
+      # @param round [Integer] the last round it ran
+      # @return [Hash, nil] the request, when there is one
+      def exhausted(path, agent, state, round)
+        plan = Plans::PlanState.for(path)
+        state = state.to_s
+        all = plan.extensions
+        request = pending(all, agent, state) ||
+                  unless asked?(all, agent, state, round, "rounds")
+                    plan.ask_extension!(agent:, state:, round:, limit: "rounds", at: @now.call)
+                  end
+        return request unless @auto && request && !auto?(all, agent, state, "round") && waited?(request)
+
+        answer!(path, request["number"], DEFAULTS.fetch("rounds"), by: AUTO)
+      end
+
+      # Settle a request with a choice. A time choice reaches a clock still
+      # running when +live+ is given; anything else is one more round, with
+      # the choice's seconds added to that round's clock.
+      #
+      # @param path [String]
+      # @param number [Integer]
+      # @param letter [String] "a", "b" or "c"
+      # @param by [String]
+      # @param live [OpenLanes::Execution::Executor::Handle, nil] the asking agent's, while it runs
+      # @return [Hash, nil] the request as settled; nil when it was settled already
+      # @raise [ArgumentError] on a letter that is not one of {CHOICES}
+      def answer!(path, number, letter, by:, live: nil)
+        choice = CHOICES.find { it.letter == letter.to_s.downcase } or
+          raise ArgumentError, "#{letter} is not one of #{CHOICES.map { "#{it.letter})" }.join(", ")}"
+        plan = Plans::PlanState.for(path)
+        request = plan.extensions.find { it["number"] == Integer(number) }
+        return nil if request.nil? || request.key?("settled_at")
+
+        # Only a time request moves a running clock: every choice on a
+        # rounds request is a round, even while the agent was resumed.
+        grant = live && request["limit"] == "time" && choice.seconds.positive? ? "live" : "round"
+        settled = plan.settle_extension!(number, answer: choice.letter, by:, grant:, seconds: choice.seconds, at: @now.call)
+        # Someone else settled it between the read above and this write.
+        return nil unless settled
+
+        # The clock told the agent STOP, or stopped, before the time got
+        # there: the time comes with a round instead of going nowhere.
+        settled = plan.amend_extension!(number, grant: "round") if grant == "live" && !live.extend!(choice.seconds)
+        UI.log("#{by == AUTO ? "auto-extended" : "extended by #{by}"}: #{request["agent"]} #{choice.label.downcase}",
+          plan: File.basename(path)[/\A\d{3}\.\d{2}/])
+        settled
+      end
+
+      # The rounds a person or this class has added to an agent's own.
+      #
+      # @param path [String]
+      # @param agent [String]
+      # @param state [#to_s]
+      # @return [Integer]
+      def rounds(path, agent, state)
+        granted(Plans::PlanState.for(path).extensions, agent, state.to_s).count { Time.iso8601(it["settled_at"]) >= @since.floor }
+      end
+
+      # The extra seconds a granted round runs on, handed out once.
+      #
+      # @param path [String]
+      # @param agent [String]
+      # @param state [#to_s]
+      # @return [Integer] 0 when the round has none
+      def take_seconds(path, agent, state)
+        plan = Plans::PlanState.for(path)
+        grant = granted(plan.extensions, agent, state.to_s).find { it["seconds"].to_i.positive? && !it["spent"] }
+        return 0 unless grant
+
+        plan.spend_extension!(grant["number"])
+        grant["seconds"]
+      end
+
+      # A request nobody needs any more: the agent's round ended, however it
+      # ended. A time card left open would say "running out of time" about an
+      # agent no longer running, and stand in for the next round's own ask.
+      #
+      # @param path [String]
+      # @param agent [String]
+      # @param state [#to_s]
+      # @return [void]
+      def close!(path, agent, state, why: "finished")
+        plan = Plans::PlanState.for(path)
+        request = pending(plan.extensions, agent, state.to_s) or return
+        plan.settle_extension!(request["number"], closed: why, at: @now.call)
+      end
+
+      class << self
+        # The requests a plan still has open for its state, as the page's
+        # `Question`s, beside the ones `blocked.md` asks.
+        #
+        # @param subject [OpenLanes::Plans::Subject]
+        # @param plan [OpenLanes::Plans::PlanState]
+        # @return [Array<Hash>]
+        def questions(subject, plan)
+          state = subject.status.key.to_s
+          plan.extensions.reject { it.key?("settled_at") || it["state"] != state }.map { question(subject, it) }
+        end
+
+        private
+
+        # @return [Hash]
+        def question(subject, request)
+          time = request["limit"] == "time"
+          recommended = DEFAULTS.fetch(request["limit"])
+          { id:       "#{subject.feature.ordinal}/X#{request["number"]}",
+            number:   request["number"],
+            kind:     "extension",
+            heading:  time ? "#{request["agent"]} is running out of time" : "#{request["agent"]} is out of rounds",
+            body:     body(subject, request, time),
+            agent:    request["agent"],
+            owner:    "technical",
+            options:  CHOICES.map do |choice|
+              { letter:      choice.letter,
+                label:       choice.label,
+                recommended: choice.letter == recommended,
+                answer:      "#{choice.letter}) #{choice.label}" }
+            end,
+            held:     false,
+            asked_at: request["asked_at"] }
+        end
+
+        # @return [String]
+        def body(subject, request, time)
+          if time
+            "Round #{request["round"]} on #{subject.status} has under #{ASK_AT / 60} minutes left on its clock. " \
+              "More time lets it carry on where it is; another round starts it again from its resume note."
+          else
+            "It has used every round it had on #{subject.status} without finishing. " \
+              "Each choice gives it one more round; the time choices also lengthen that round's clock."
+          end
+        end
+      end
+
+      private
+
+      # @return [Hash, nil] the agent's unsettled request on this state
+      def pending(all, agent, state) = all.find { it["agent"] == agent && it["state"] == state && !it.key?("settled_at") }
+
+      # @return [Array<Hash>] the requests that gave the agent a round on this state
+      def granted(all, agent, state) = all.select { it["grant"] == "round" && it["agent"] == agent && it["state"] == state }
+
+      # A time request answered with time on the running clock does not
+      # count: when that time runs low too, the round asks again, so a
+      # person can add to an automatic grant.
+      #
+      # @return [Boolean] whether this round already asked about this limit
+      def asked?(all, agent, state, round, limit)
+        all.any? do |r|
+          r["agent"] == agent && r["state"] == state && r["round"] == round && r["limit"] == limit && r["grant"] != "live"
+        end
+      end
+
+      # @return [Boolean] whether the one grant of this kind nobody answered is spent
+      def auto?(all, agent, state, grant)
+        all.any? { it["agent"] == agent && it["state"] == state && it["by"] == AUTO && it["grant"] == grant }
+      end
+
+      # @return [Boolean] whether a rounds request has waited long enough for a person
+      def waited?(request)
+        return true unless @stay
+
+        @now.call - Time.iso8601(request["asked_at"]) >= AUTO_AFTER
+      end
+    end
+  end
+end