openlanes 3.0.0

data/lib/openlanes/live/memory_store.rb

@@ -0,0 +1,265 @@
+# frozen_string_literal: true
+
+require "monitor"
+
+module OpenLanes
+  module Live
+    # {Store}'s methods, kept in this process. The suite's Redis.
+    #
+    # It emulates exactly as much of Redis as the contract uses, and the
+    # shared examples in `spec/openlanes/live/store_examples.rb` hold it to
+    # the real thing: an expiry read off a clock the example controls,
+    # compare-and-delete, a consumer group's pending list, and approximate
+    # trimming done exactly (which `MAXLEN ~` permits: it promises at least
+    # that many, never exactly that many).
+    class MemoryStore
+      # @param clock [Proc] seconds, monotonic; an example moves it to expire keys
+      def initialize(clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
+        @clock = clock
+        @mutex = Monitor.new
+        @data = {}
+        @expiry = {}
+        @groups = Hash.new { |h, k| h[k] = {} }
+        @last_ms = 0
+        @last_seq = 0
+      end
+
+      # @return [String] what {Live.mask} shows for it
+      def url = "memory://"
+
+      # @return [Boolean]
+      def ping = true
+
+      # @return [void]
+      def close = nil
+
+      # ---- strings ----------------------------------------------------------
+
+      # @return [String, nil]
+      def get(key) = locked { live(key) }
+
+      # @return [Boolean] whether the value was written
+      def set(key, value, ex: nil, nx: false)
+        locked do
+          next false if nx && !live(key).nil?
+
+          @data[key] = value.to_s
+          ex ? @expiry[key] = now + ex : @expiry.delete(key)
+          true
+        end
+      end
+
+      # @return [Integer] how many were removed
+      def del(*keys) = locked { keys.count { |key| forget(key) } }
+
+      # @return [Boolean] whether the key exists to expire
+      def expire(key, seconds)
+        locked do
+          next false if live(key).nil?
+
+          @expiry[key] = now + seconds
+          true
+        end
+      end
+
+      # @return [Integer] seconds left, -1 without an expiry, -2 when absent
+      def ttl(key)
+        locked do
+          next -2 if live(key).nil?
+
+          @expiry.key?(key) ? (@expiry[key] - now).ceil : -1
+        end
+      end
+
+      # @return [Boolean]
+      def compare_and_delete(key, expected)
+        locked { live(key) == expected ? forget(key) : false }
+      end
+
+      # @return [Boolean]
+      def compare_and_expire(key, expected, seconds)
+        locked { live(key) == expected ? expire(key, seconds) : false }
+      end
+
+      # ---- hashes -----------------------------------------------------------
+
+      # @return [Boolean] whether the field is new
+      def hset(key, field, value)
+        locked do
+          hash = live(key) || (@data[key] = {})
+          fresh = !hash.key?(field.to_s)
+          hash[field.to_s] = value.to_s
+          fresh
+        end
+      end
+
+      # @return [String, nil]
+      def hget(key, field) = locked { live(key)&.fetch(field.to_s, nil) }
+
+      # @return [Hash{String => String}]
+      def hgetall(key) = locked { (live(key) || {}).dup }
+
+      # @return [Integer]
+      def hdel(key, *fields)
+        locked do
+          hash = live(key) or next 0
+          removed = fields.count { |f| hash.delete(f.to_s) }
+          forget(key) if hash.empty?
+          removed
+        end
+      end
+
+      # ---- streams ----------------------------------------------------------
+
+      # @return [String] the new entry's id
+      def xadd(key, fields, maxlen: nil)
+        locked do
+          stream = live(key) || (@data[key] = [])
+          id = next_id
+          stream << Store::Entry.new(id:, fields: fields.to_h { |k, v| [k.to_s, v.to_s] })
+          stream.shift(stream.size - maxlen) if maxlen && stream.size > maxlen
+          id
+        end
+      end
+
+      # @return [Integer]
+      def xlen(key) = locked { (live(key) || []).size }
+
+      # @return [Boolean] whether the group is new
+      def xgroup_create(key, group, from: "0")
+        locked do
+          stream = live(key) || (@data[key] = [])
+          next false if @groups[key].key?(group)
+
+          last = from == "$" ? stream.last&.id || "0-0" : from
+          @groups[key][group] = { last: id_of(last.include?("-") ? last : "#{last}-0"), pending: {} }
+          true
+        end
+      end
+
+      # @return [Array<Store::Entry>]
+      def xreadgroup(key, group, consumer, id: ">", count: 100)
+        locked do
+          state = @groups[key][group] or raise Unavailable, "NOGROUP #{group} on #{key}"
+          stream = live(key) || []
+          if id == ">"
+            fresh = stream.select { |e| (id_of(e.id) <=> state[:last]).positive? }.first(count)
+            fresh.each { |e| state[:pending][e.id] = consumer }
+            state[:last] = id_of(fresh.last.id) unless fresh.empty?
+            fresh
+          else
+            floor = id_of(id.include?("-") ? id : "#{id}-0")
+            state[:pending].select { |pid, who| who == consumer && (id_of(pid) <=> floor).positive? }.keys
+                           .sort_by { id_of(it) }.first(count)
+                           .map { |pid| stream.find { |e| e.id == pid } || Store::Entry.new(id: pid, fields: {}) }
+          end
+        end
+      end
+
+      # @return [Integer]
+      def xack(key, group, *ids)
+        locked do
+          pending = @groups.dig(key, group, :pending) or next 0
+          ids.count { |id| pending.delete(id) }
+        end
+      end
+
+      # Never waits: nothing else can add to this process's streams while
+      # the caller is blocked, so +block+ is accepted and ignored.
+      #
+      # @return [Array<Store::Entry>]
+      def xread(key, id: "0", count: 100, block: nil) # rubocop:disable Lint/UnusedMethodArgument
+        locked do
+          stream = live(key) || []
+          next [] if id == "$"
+
+          floor = id_of(id.include?("-") ? id : "#{id}-0")
+          stream.select { |e| (id_of(e.id) <=> floor).positive? }.first(count)
+        end
+      end
+
+      # @return [Array<Store::Entry>] oldest first
+      def xrange(key, start: "-", stop: "+", count: 100)
+        locked { within(live(key) || [], start, stop).first(count) }
+      end
+
+      # @return [Array<Store::Entry>] newest first
+      def xrevrange(key, stop: "+", start: "-", count: 100)
+        locked { within(live(key) || [], start, stop).reverse.first(count) }
+      end
+
+      # ---- keys -------------------------------------------------------------
+
+      # @return [Array<String>]
+      def scan(prefix) = locked { @data.keys.select { |k| k.start_with?(prefix) && !live(k).nil? }.sort }
+
+      # ---- transactions -----------------------------------------------------
+
+      # @yieldparam tx [Store::Transaction]
+      # @return [void]
+      def multi
+        tx = Store::Transaction.new
+        yield tx
+        locked { tx.calls.each { |name, args, kwargs| public_send(name, *args, **kwargs) } }
+        nil
+      end
+
+      private
+
+      # @return [Object] the block's value, under the lock
+      def locked(&) = @mutex.synchronize(&)
+
+      # @return [Float]
+      def now = @clock.call
+
+      # @return [Object, nil] the value, unless it has expired
+      def live(key)
+        if @expiry.key?(key) && @expiry[key] <= now
+          forget(key)
+          return nil
+        end
+        @data[key]
+      end
+
+      # @return [Boolean] whether there was anything to forget
+      def forget(key)
+        @expiry.delete(key)
+        @groups.delete(key)
+        !@data.delete(key).nil?
+      end
+
+      # Ids as Redis makes them: milliseconds, then a sequence within the
+      # millisecond, always increasing even if the clock does not.
+      #
+      # @return [String]
+      def next_id
+        ms = (Time.now.to_f * 1000).to_i
+        if ms > @last_ms
+          @last_ms = ms
+          @last_seq = 0
+        else
+          @last_seq += 1
+        end
+        "#{@last_ms}-#{@last_seq}"
+      end
+
+      # @return [Array(Integer, Integer)]
+      def id_of(id) = id.split("-", 2).map(&:to_i)
+
+      # @return [Array<Store::Entry>]
+      def within(stream, start, stop)
+        low = if start == "-"
+                [0, 0]
+              else
+                id_of(start.include?("-") ? start : "#{start}-0")
+              end
+        high = if stop == "+"
+                 [Float::INFINITY, Float::INFINITY]
+               else
+                 id_of(stop.include?("-") ? stop : "#{stop}-#{2**63}")
+               end
+        stream.select { (low..high).cover?(id_of(it.id)) }
+      end
+    end
+  end
+end

data/lib/openlanes/live/meter.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Live
+    # What each plan has spent this run, kept across the board's rows.
+    #
+    # A row leaves the board ten seconds after its agent finishes, and what it
+    # spent must not leave the plan's total with it. So every row seen is
+    # remembered by plan, agent, round and invocation, at the highest figures
+    # it showed. The invocation matters because switch_model starts an agent
+    # again in the same round, on another model: keyed by round alone, the
+    # restart's smaller figures would hide the first run's spend, and the
+    # whole round would be priced at the second model.
+    class Meter
+      # One invocation of an agent's round on one plan, at its latest reading.
+      Spend = Data.define(:up, :down, :model)
+
+      def initialize
+        @spends = Hash.new { |h, k| h[k] = {} }
+        @first_seen = {}
+      end
+
+      # @param rows [Array<OpenLanes::Engine::Board::Row>]
+      # @param now [Time]
+      # @return [void]
+      def record(rows, now: Time.now)
+        rows.each do |row|
+          @first_seen[row.ordinal] ||= now
+          key = [row.agent, row.round, row.since]
+          known = @spends[row.ordinal][key]
+          @spends[row.ordinal][key] =
+            Spend.new(up: [row.up.to_i, known&.up.to_i].max, down: [row.down.to_i, known&.down.to_i].max, model: row.model)
+        end
+      end
+
+      # @param ordinal [String]
+      # @return [Time, nil] when this run first saw an agent on the plan
+      def started_at(ordinal) = @first_seen[ordinal]
+
+      # @param ordinal [String]
+      # @param price [#call] model name to an {Adapters::Model}, or nil
+      # @return [Hash{Symbol => Numeric}] `up`, `down` and `cost`
+      def for(ordinal, price)
+        spends = @spends.fetch(ordinal, {}).values
+        { up:   spends.sum(&:up),
+          down: spends.sum(&:down),
+          cost: spends.sum { |s| price.call(s.model)&.cost(s.up, s.down).to_f }.round(4) }
+      end
+    end
+  end
+end

data/lib/openlanes/live/node.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require "digest"
+require "etc"
+require "socket"
+
+module OpenLanes
+  module Live
+    # This machine and user, as other machines sharing the Redis see it (W26).
+    #
+    # The id is a digest of `user@host` so keys are short and uniform, not to
+    # hide anyone: the plain names travel beside it. The checkout id keeps
+    # two clones on one machine apart.
+    #
+    # @!attribute [r] id
+    #   @return [String] 24 hex digits of SHA-256 of `user@host`
+    # @!attribute [r] user
+    #   @return [String]
+    # @!attribute [r] host
+    #   @return [String]
+    # @!attribute [r] ip
+    #   @return [String, nil] shown, never used to identify the node
+    # @!attribute [r] checkout
+    #   @return [String] the checkout's path, `$HOME` written as `~`
+    # @!attribute [r] checkout_id
+    #   @return [String] 12 hex digits of SHA-256 of the checkout's realpath
+    Node = Data.define(:id, :user, :host, :ip, :checkout, :checkout_id) do
+      # @return [String] `<id>:<checkout_id>`, what keys and claims carry
+      def name = "#{id}:#{checkout_id}"
+
+      # @return [String] `user@host`, what the page shows
+      def label = "#{user}@#{host}"
+
+      # The `Node` of `web/src/api/types.ts`.
+      #
+      # @param heartbeat_at [Time]
+      # @param me [Boolean]
+      # @return [Hash]
+      def wire(heartbeat_at:, me: true)
+        { id:, user:, host:, ip:, checkout:, heartbeat_at: heartbeat_at.utc.iso8601, me: }
+      end
+    end
+
+    class Node
+      # Hex digits kept of each digest.
+      ID_LENGTH = 24
+      CHECKOUT_LENGTH = 12
+
+      class << self
+        # @param root [String] the checkout
+        # @param user [String]
+        # @param host [String]
+        # @param home [String]
+        # @return [OpenLanes::Live::Node]
+        def current(root:, user: login, host: Socket.gethostname, home: Dir.home, ip: address)
+          real = File.realpath(root)
+          new(id: digest("#{user}@#{host}", ID_LENGTH),
+            user:,
+            host:,
+            ip:,
+            checkout: tilde(real, home),
+            checkout_id: digest(real, CHECKOUT_LENGTH))
+        end
+
+        # @param text [String]
+        # @param length [Integer]
+        # @return [String]
+        def digest(text, length) = Digest::SHA256.hexdigest(text)[0, length]
+
+        # @return [String]
+        def login = Etc.getlogin || ENV.fetch("USER", "unknown")
+
+        # The first address that is neither loopback nor link-local: the one
+        # other machines on the network most likely reach this one by.
+        #
+        # @return [String, nil]
+        def address
+          Socket.ip_address_list.find { |a| a.ipv4? && !a.ipv4_loopback? && !a.ipv4_multicast? }&.ip_address
+        end
+
+        # @param path [String]
+        # @param home [String]
+        # @return [String]
+        def tilde(path, home) = path == home || path.start_with?("#{home}/") ? "~#{path.delete_prefix(home)}" : path
+      end
+    end
+  end
+end

data/lib/openlanes/live/publisher.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require "json"
+
+module OpenLanes
+  module Live
+    # The engine's side of the board (W13): handed each tick's
+    # {Engine::Board} as the runner's `on_board`, it writes the snapshot,
+    # appends it to the board stream, refreshes this node's heartbeat and
+    # renews its claims.
+    #
+    # It also keeps the latest log lines for the page, by tapping {UI.log}
+    # rather than asking callers to log twice.
+    #
+    # A Redis that goes away mid-run is reported once and otherwise
+    # ignored: the run is the agents' work, and the page is a view of it.
+    class Publisher
+      # Log lines a snapshot carries.
+      LOG_LINES = 200
+
+      # Words in a log line that mark it as something gone wrong.
+      PROBLEMS = /\b(?:fail(?:ed|ure)?|error|refused|unavailable|died)\b/i
+
+      # @param store [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore]
+      # @param keys [OpenLanes::Live::Keys]
+      # @param node [OpenLanes::Live::Node]
+      # @param tree [OpenLanes::Plans::Tree]
+      # @param agents [OpenLanes::Agents::Registry]
+      # @param project [String]
+      # @param claims [OpenLanes::Live::Claims, nil]
+      # @param run_state [OpenLanes::Live::RunState]
+      # @param description [String, nil]
+      # @param clock [Proc] wall time
+      def initialize(store:, keys:, node:, tree:, agents:, project:, claims: nil, run_state: RunState.new,
+        description: nil, clock: -> { Time.now })
+        @store = store
+        @keys = keys
+        @node = node
+        @tree = tree
+        @agents = agents
+        @project = project
+        @claims = claims
+        @run_state = run_state
+        @description = description
+        @clock = clock
+        @meter = Meter.new
+        @log = []
+        @log_mutex = Mutex.new
+        @seq = nil
+        @warned = false
+        @on_lost = nil
+      end
+
+      # What to do with plans whose claim lapsed and went to another node:
+      # the session stops their agents, so two machines never work one plan.
+      #
+      # @yieldparam ordinals [Array<String>]
+      # @return [void]
+      def on_lost(&block)
+        @on_lost = block
+      end
+
+      # @return [OpenLanes::Live::Meter]
+      attr_reader :meter
+
+      # @return [Integer, nil] the last sequence number published
+      attr_reader :seq
+
+      # @return [Proc] for {Engine::Runner}'s `on_board:`
+      def to_proc = method(:call).to_proc
+
+      # @param board [OpenLanes::Engine::Board]
+      # @return [Hash, nil] the snapshot published, nil when Redis was not there
+      def call(board)
+        now = @clock.call
+        @started_at ||= now - (UI.monotonic - board.started_at)
+        @meter.record(board.rows, now:)
+        snapshot = build(board, now)
+        json = JSON.generate(snapshot)
+        @store.multi do |tx|
+          tx.set(@keys.snapshot, json, ex: Keys::SNAPSHOT_TTL)
+          tx.xadd(@keys.board, { seq: snapshot[:seq], json: }, maxlen: Keys::BOARD_LENGTH)
+          tx.hset(@keys.nodes, @node.name, JSON.generate(@node.wire(heartbeat_at: now)))
+        end
+        lost = @claims&.renew || []
+        unless lost.empty?
+          UI.log("claims lost to another machine: #{lost.join(", ")}; stopping their agents")
+          @on_lost&.call(lost)
+        end
+        @warned = false
+        snapshot
+      rescue Unavailable => e
+        unavailable(e)
+        nil
+      end
+
+      # Start keeping {UI.log}'s lines.
+      #
+      # @return [void]
+      def listen
+        return if @tap
+
+        @tap = UI.tap_log(method(:hear))
+      end
+
+      # Stop keeping them.
+      #
+      # @return [void]
+      def close
+        UI.untap_log(@tap) if @tap
+        @tap = nil
+      end
+
+      # One log line, as the page shows it. Public so a component that does
+      # not go through {UI.log} can still be heard.
+      #
+      # @param message [String]
+      # @param fields [Hash] `plan:`, `agent:` and the rest of {UI.log}'s columns
+      # @param source [String, nil]
+      # @param level [String, nil]
+      # @return [void]
+      def hear(message, fields = {}, source: nil, level: nil)
+        line = { at:     @clock.call.utc.iso8601,
+                 source: source || fields[:agent]&.to_s || "engine",
+                 level:  level || (message.to_s.match?(PROBLEMS) ? "warn" : "info"),
+                 text:   message.to_s,
+                 plan:   fields[:plan]&.to_s }
+        @log_mutex.synchronize do
+          @log << line
+          @log.shift(@log.size - LOG_LINES) if @log.size > LOG_LINES
+        end
+      end
+
+      # @return [Array<Hash>] the lines kept, newest last
+      def log = @log_mutex.synchronize { @log.dup }
+
+      private
+
+      # @param board [OpenLanes::Engine::Board]
+      # @param now [Time]
+      # @return [Hash]
+      def build(board, now)
+        Snapshot.new(board:,
+          tree:        @tree,
+          agents:      @agents,
+          node:        @node,
+          repo:        @keys.repo,
+          project:     @project,
+          description: @description,
+          seq:         next_seq,
+          at:          now,
+          started_at:  @started_at,
+          claims:      @claims,
+          run_state:   @run_state,
+          meter:       @meter,
+          log:).to_h
+      end
+
+      # One more than the last published, even across an engine restart:
+      # the API server reads a change of `seq` as news, and a restart that
+      # counted from one again would look like the past.
+      #
+      # @return [Integer]
+      def next_seq
+        @seq ||= @store.xrevrange(@keys.board, count: 1).first&.fields&.fetch("seq", nil).to_i
+        @seq += 1
+      end
+
+      # @param error [OpenLanes::Live::Unavailable]
+      # @return [void]
+      def unavailable(error)
+        return if @warned
+
+        @warned = true
+        UI.log("board not published: #{error.message}")
+      end
+    end
+  end
+end

data/lib/openlanes/live/repo.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "digest"
+require "open3"
+
+module OpenLanes
+  module Live
+    # The repository a run belongs to, as the part of every key that machines
+    # sharing one Redis have in common (W26, W29): `github.com/<owner>/<name>`.
+    #
+    # Readable rather than hashed, so `redis-cli --scan` shows whose
+    # repository a key belongs to. The clone decides which factory a machine
+    # joins, and nothing else does, which is why there is no override.
+    module Repo
+      module_function
+
+      # `git@github.com:Owner/Name.git`, `https://github.com/owner/name`,
+      # `ssh://git@github.com/owner/name.git` and their trailing slashes and
+      # letter case all fold to one spelling.
+      #
+      # @param url [String]
+      # @return [String, nil] `host/owner/name`, nil for a string that is not
+      #   a remote URL
+      def normalise(url)
+        text = url.to_s.strip.downcase.delete_suffix("/").delete_suffix(".git")
+        path = case text
+               when %r{\A[a-z][a-z0-9+.-]*://(?:[^@/]+@)?([^/:]+)(?::\d+)?/(.+)\z},
+                    %r{\A(?:[^@/]+@)?([^/:]+):(.+)\z} then "#{::Regexp.last_match(1)}/#{::Regexp.last_match(2)}"
+               end
+        path&.squeeze("/")
+      end
+
+      # The repository of the checkout at +root+: its `origin`, normalised,
+      # or, with no remote to share, `local/<name>-<12 hex>`, the directory's
+      # name and a digest of its real path. The name alone collided: two
+      # unrelated clones both called `app` on one Redis would have read each
+      # other's boards and taken each other's claims.
+      #
+      # @param root [String]
+      # @return [String]
+      def for(root)
+        out, status = Open3.capture2("git", "-C", root, "remote", "get-url", "origin", err: File::NULL)
+        (status.success? && normalise(out)) || local(root)
+      rescue SystemCallError
+        local(root)
+      end
+
+      # @param root [String]
+      # @return [String] `local/<basename>-<12 hex of SHA-256 of the real path>`
+      def local(root)
+        path = begin
+          File.realpath(root)
+        rescue SystemCallError
+          File.expand_path(root)
+        end
+        "local/#{File.basename(path)}-#{Digest::SHA256.hexdigest(path)[0, 12]}"
+      end
+    end
+  end
+end