openlanes 3.0.1 → 3.1.0

data/lib/openlanes/plans/journal.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require "json"
+
+module OpenLanes
+  module Plans
+    # An agent's journal (M2): what it decided, did, left to do, asked and
+    # must not forget, kept from one round to the next.
+    #
+    # The agent writes no file and runs no command. It prints each entry as
+    # one line of JSON in its own output, which the harness reads anyway, and
+    # the dispatcher keeps it in `state.json` under `memory.<agent>`. One
+    # writer, under the lock {PlanState} already takes, is what lets a pair
+    # journal at once without either losing the other's line; an agent that
+    # wrote the file itself would be a second writer racing the first.
+    #
+    #     {"openlanes_memory": {"kind": "decision", "text": "Money is integer cents"}}
+    #     {"openlanes_memory": {"kind": "override", "text": "BigDecimal after all", "overrides": 3}}
+    #
+    # The single key {KEY} is what tells an entry from any other JSON an
+    # agent prints: a status object, a `package.json` it quotes, a `jq`
+    # result. Nothing else an agent has reason to print starts with that key.
+    #
+    # There is no timestamp. Entries are numbered in order across the
+    # agent's rounds on the plan, and a later one wins over an earlier one;
+    # an `override` names the number it overturns.
+    #
+    # Pure: text in, entries out. The dispatcher decides where they go.
+    module Journal
+      # The one key an entry line carries.
+      KEY = "openlanes_memory"
+
+      # What an entry may be. `question` is the agent stopping to ask; the
+      # harness adds the answer beside it once one arrives (M3).
+      KINDS = %w[todo done decision question override note].freeze
+
+      # Longest text kept, in characters. Every round's prompt carries the
+      # journal, so an entry is a line to remember, not a document; one that
+      # runs on is cut rather than refused, since what it says first is what
+      # it most needed to keep.
+      TEXT_LIMIT = 500
+
+      # A line that may be an entry: the key first, after an optional
+      # backtick, since agents quote JSON in Markdown as often as not.
+      CANDIDATE = /\A`*\{\s*"#{KEY}"\s*:/
+
+      # One entry, as the agent printed it or the harness recorded it.
+      #
+      # @!attribute [r] kind
+      #   @return [String] one of {KINDS}
+      # @!attribute [r] text
+      #   @return [String]
+      # @!attribute [r] overrides
+      #   @return [Integer, nil] the number of the entry an `override` overturns
+      # @!attribute [r] ref
+      #   @return [String, nil] `B<n>`, for a question the harness recorded
+      #     from `blocked.md`
+      # @!attribute [r] answer
+      #   @return [String, nil] a person's answer to that question
+      Entry = Data.define(:kind, :text, :overrides, :ref, :answer) do
+        def initialize(kind:, text:, overrides: nil, ref: nil, answer: nil) = super
+
+        # @param hash [Hash] as `state.json` keeps it
+        # @return [OpenLanes::Plans::Journal::Entry]
+        def self.from_h(hash)
+          new(kind: hash["kind"].to_s,
+            text: hash["text"].to_s,
+            overrides: hash["overrides"],
+            ref: hash["ref"],
+            answer: hash["answer"])
+        end
+
+        # @return [Hash] string keys, nils left out, as the schema wants
+        def to_json_h = to_h.transform_keys(&:to_s).compact
+      end
+
+      # A line that announced itself as an entry and was not one, kept so the
+      # dispatcher can say so: an agent whose journal is silently dropped
+      # believes it is remembering.
+      #
+      # @!attribute [r] line
+      #   @return [String]
+      # @!attribute [r] reason
+      #   @return [String]
+      Rejection = Data.define(:line, :reason)
+
+      # One round's worth of one agent's journal.
+      #
+      # @!attribute [r] number
+      #   @return [Integer] the agent's journal round on this plan, 1 up,
+      #     counted across runs and states. Not the dispatcher's round, which
+      #     restarts at 1 in every run and every state and would land a later
+      #     run's entries on an earlier run's.
+      # @!attribute [r] round
+      #   @return [Integer, nil] the dispatcher's round it was printed in
+      # @!attribute [r] state
+      #   @return [String, nil] the state the plan was in when it started
+      # @!attribute [r] entries
+      #   @return [Array<Entry>]
+      # @!attribute [r] first
+      #   @return [Integer] the number of its first entry in the whole journal
+      Round = Data.define(:number, :round, :state, :entries, :first)
+
+      module_function
+
+      # Every entry line in one piece of the agent's text, in order, each
+      # read into an {Entry} or a {Rejection}. Lines that do not start with
+      # {KEY} are none of this method's business and are skipped.
+      #
+      # @param text [String, nil] what the agent said
+      # @return [Array<Entry, Rejection>]
+      def scan(text)
+        text.to_s.each_line.filter_map do |raw|
+          line = raw.strip
+          read(line) if line.match?(CANDIDATE)
+        end
+      end
+
+      # @param line [String] one candidate line
+      # @return [Entry, Rejection]
+      def read(line)
+        json = line.delete_prefix("`" * line[/\A`*/].size).delete_suffix("`" * line[/`*\z/].size)
+        value = JSON.parse(json)[KEY]
+        problem = problem_with(value)
+        return Rejection.new(line:, reason: problem) if problem
+
+        Entry.new(kind: value["kind"], text: clip(value["text"]), overrides: value["overrides"])
+      rescue JSON::ParserError => e
+        Rejection.new(line:, reason: "not JSON: #{e.message.lines.first.to_s.strip}")
+      end
+
+      # Only the fields an agent may set are read; `ref` and `answer` are the
+      # harness's to write, and an agent claiming a person answered something
+      # is exactly the line that must not be trusted.
+      #
+      # @param value [Object]
+      # @return [String, nil] why it is not an entry
+      def problem_with(value)
+        return "#{KEY} is not an object" unless value.is_a?(Hash)
+        return "kind #{value["kind"].inspect} is not one of #{KINDS.join(", ")}" unless KINDS.include?(value["kind"])
+        return "text is missing" unless value["text"].is_a?(String) && !value["text"].strip.empty?
+
+        overrides = value["overrides"]
+        return nil if overrides.nil?
+        return "overrides must be an entry number" unless overrides.is_a?(Integer) && overrides.positive?
+
+        "only an override may name an entry it overrides" unless value["kind"] == "override"
+      end
+
+      # @param text [String]
+      # @return [String] stripped, and cut to {TEXT_LIMIT}
+      def clip(text)
+        text = text.to_s.strip
+        text.length > TEXT_LIMIT ? "#{text[0, TEXT_LIMIT - 1]}…" : text
+      end
+
+      # The line an agent prints, for the prompt's examples, so the prompt and
+      # the parser cannot describe two shapes.
+      #
+      # @param kind [String]
+      # @param text [String]
+      # @param overrides [Integer, nil]
+      # @return [String]
+      def line(kind, text, overrides: nil)
+        JSON.generate(KEY => { "kind" => kind, "text" => text, "overrides" => overrides }.compact)
+      end
+    end
+  end
+end

data/lib/openlanes/plans/last_known_state.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "time"
+
+module OpenLanes
+  module Plans
+    # What an interrupted agent had done, as a record the next round acts on.
+    #
+    # It replaces the `RESUME: ...` mail, which was prose the next agent had
+    # to read and believe. The fields are fixed: freeform JSON would be prose
+    # with braces, and no easier to act on.
+    #
+    # Two writers. The agent, through `openlanes state resume`, when it
+    # reaches a polling point after INTERRUPT. The harness, when the agent
+    # never did: killed when GRACE ran out, or dead outright. The agent knows
+    # what it did, the harness only what it saw, so an agent's entry is never
+    # replaced by a harness one from the same round or an earlier one.
+    #
+    # Pids, tokens and exit codes are not here and cannot be: this lands in
+    # the committed `state.json`, and process detail belongs to one machine's
+    # `openlanes-state.json`.
+    #
+    # @!attribute [r] source
+    #   @return [String] "agent" or "harness"
+    # @!attribute [r] at
+    #   @return [String] ISO 8601
+    # @!attribute [r] round
+    #   @return [Integer]
+    # @!attribute [r] status
+    #   @return [String] one of {Ledger::STATUSES}
+    # @!attribute [r] done
+    #   @return [Array<String>]
+    # @!attribute [r] remaining
+    #   @return [Array<String>]
+    # @!attribute [r] next_step
+    #   @return [String, nil] the one thing to do first
+    # @!attribute [r] files_touched
+    #   @return [Array<String>]
+    # @!attribute [r] note
+    #   @return [String, nil] what the harness saw, for a harness entry
+    LastKnownState = Data.define(:source, :at, :round, :status, :done, :remaining, :next_step, :files_touched, :note) do
+      # @return [Array<String>]
+      def self.sources = %w[agent harness]
+
+      # @param hash [Hash] as `state.json` holds it
+      # @return [OpenLanes::Plans::LastKnownState]
+      def self.from_h(hash)
+        new(**members.to_h { |k| [k, hash[k.to_s]] }.compact)
+      end
+
+      # @raise [OpenLanes::Error] for a source or status outside the vocabulary
+      def initialize(source:, round:, status:, at: Time.now.iso8601, done: [], remaining: [], next_step: nil,
+        files_touched: [], note: nil)
+        raise Error, "source must be one of #{self.class.sources.join(", ")}" unless self.class.sources.include?(source)
+        raise Error, "status must be one of #{Ledger::STATUSES.join(", ")}" unless Ledger::STATUSES.include?(status)
+
+        super(source:, at: at.respond_to?(:iso8601) ? at.iso8601 : at.to_s, round: Integer(round), status:,
+          done: Array(done), remaining: Array(remaining), next_step:, files_touched: Array(files_touched), note:)
+      end
+
+      # @return [Boolean]
+      def agent? = source == "agent"
+
+      # Whether this entry may replace +other+, the one already kept. An
+      # agent's entry replaces anything; the harness's only what is not an
+      # agent's of the same round or later.
+      #
+      # @param other [OpenLanes::Plans::LastKnownState, nil]
+      # @return [Boolean]
+      def supersedes?(other)
+        return true if other.nil? || agent?
+
+        !(other.agent? && other.round >= round)
+      end
+
+      # @return [Hash] as `state.json` holds it, empty fields left out
+      def to_json_h
+        to_h.transform_keys(&:to_s).reject { |_, v| v.nil? || v == [] }
+      end
+    end
+  end
+end

data/lib/openlanes/plans/lock.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require "json"
+require "socket"
+require "time"
+
+module OpenLanes
+  module Plans
+    # An agent is working on this plan right now. The folder name never
+    # changes for it, because a rename while an agent holds the old path is
+    # how its writes land in a directory that no longer exists.
+    #
+    # The file lives in the runtime state directory ({OpenLanes::Runtime}),
+    # named for the plan's ordinal, not in the plan folder: it says something
+    # true only on the machine that wrote it, and keyed by ordinal it
+    # survives a rename. A `locked-by.pid` an older release left in the
+    # folder is still read ({.legacy_path}), and removed with the lock.
+    #
+    # @!attribute [r] pid
+    #   @return [Integer]
+    # @!attribute [r] host
+    #   @return [String]
+    # @!attribute [r] started_at
+    #   @return [Time]
+    Lock = Data.define(:pid, :host, :started_at) do
+      # The plan folder's parent is the plans directory, and its parent the
+      # root {Runtime} keys by, so the harness and an agent handed the same
+      # `--dir` find one lock.
+      #
+      # @param dir [String] a plan folder
+      # @return [String] `<locks_dir>/<ordinal>.lock`
+      def self.path(dir)
+        name = File.basename(dir)
+        ordinal = Ordinal.from_dirname(name)&.to_s || name
+        File.join(Runtime.locks_dir(root: File.dirname(File.expand_path(dir), 2)), "#{ordinal}.lock")
+      end
+
+      # @param dir [String] a plan folder
+      # @return [String] where releases before the state directory wrote it
+      def self.legacy_path(dir) = File.join(dir, self::FILENAME)
+
+      # The one entry point every reader goes through: the state directory
+      # first, a legacy file second.
+      #
+      # @param dir [String] a plan folder
+      # @return [OpenLanes::Plans::Lock, nil] nil when there is no lock, or it cannot be read
+      def self.read(dir) = parse(path(dir)) || parse(legacy_path(dir))
+
+      # @param file [String]
+      # @return [OpenLanes::Plans::Lock, nil]
+      def self.parse(file)
+        data = JSON.parse(File.read(file))
+        new(pid: Integer(data.fetch("pid")), host: data.fetch("host").to_s, started_at: Time.iso8601(data.fetch("started_at")))
+      rescue Errno::ENOENT, Errno::EISDIR, JSON::ParserError, KeyError, ArgumentError, TypeError, NoMethodError
+        nil
+      end
+      private_class_method :parse
+
+      # Claim the plan for this process. Creating the file is the claim
+      # (`O_EXCL`), so two runs that both saw no lock cannot both write it
+      # and have the later one win in silence. A lock this process already
+      # holds is rewritten, which is how a pair's partner and a renamed
+      # folder find it; one left by a local process that has died is removed
+      # and the claim tried once more.
+      #
+      # @param dir [String] a plan folder
+      # @param pid [Integer]
+      # @param now [Time]
+      # @return [OpenLanes::Plans::Lock]
+      # @raise [OpenLanes::Plans::Lock::Held] when another live run holds it
+      def self.write(dir, pid: Process.pid, now: Time.now)
+        lock = new(pid:, host: Socket.gethostname, started_at: now.utc)
+        body = "#{JSON.generate(lock.to_json_hash)}\n"
+        legacy = parse(legacy_path(dir))
+        raise self::Held, "#{dir} is locked by pid #{legacy.pid} on #{legacy.host}" if legacy&.held_elsewhere? && legacy.pid != pid
+
+        2.times do
+          File.open(path(dir), File::WRONLY | File::CREAT | File::EXCL) { it.write(body) }
+          return lock
+        rescue Errno::EEXIST
+          held = read(dir)
+          if held&.local? && held.pid == pid
+            File.write(path(dir), body)
+            return lock
+          end
+          raise self::Held, "#{dir} is locked by pid #{held.pid} on #{held.host}" if held && !held.stale?
+          raise self::Held, "#{dir} has a lock being written or unreadable" if held.nil? && fresh?(dir)
+
+          remove(dir)
+        end
+        raise self::Held, "#{dir}: another run claimed it first"
+      end
+
+      # @param dir [String]
+      # @return [Boolean] an unreadable lock young enough to be one another
+      #   run has just created and not yet filled
+      def self.fresh?(dir)
+        Time.now - File.mtime(path(dir)) < self::UNREADABLE_GRACE
+      rescue Errno::ENOENT
+        false
+      end
+      private_class_method :fresh?
+
+      # Also clears a legacy file, but only one a dead process on this host
+      # left: that is how `resync dirs` keeps clearing old stale locks, and a
+      # live or foreign one in the folder is somebody else's claim.
+      #
+      # @param dir [String] a plan folder
+      # @return [void]
+      def self.remove(dir)
+        FileUtils.rm_f(path(dir))
+        legacy = parse(legacy_path(dir))
+        FileUtils.rm_f(legacy_path(dir)) if legacy&.stale?
+      end
+
+      # @return [Boolean] whether this machine wrote it, so its pid means something here
+      def local? = host == Socket.gethostname
+
+      # Written on another host. Held for as long as it exists: never stale,
+      # never cleared here, since nothing on this machine can tell it is over.
+      #
+      # @return [Boolean]
+      def foreign? = !local?
+
+      # Only a local lock can be checked; one from another host is never
+      # declared dead, since its pid names a process on a machine we cannot see.
+      #
+      # @return [Boolean]
+      def alive?
+        return false unless local?
+
+        Process.kill(0, pid)
+        true
+      rescue Errno::ESRCH
+        false
+      rescue Errno::EPERM
+        true
+      end
+
+      # @return [Boolean] written on this machine by a process that is gone
+      def stale? = local? && !alive?
+
+      # @return [Boolean] written by this very process, as a pair's partner finds it
+      def ours? = local? && pid == Process.pid
+
+      # Another run is in this folder: a live local process that is not this
+      # one, or any process on another machine. Such a folder is neither
+      # dispatched nor renamed here.
+      #
+      # @return [Boolean]
+      def held_elsewhere? = !ours? && !stale?
+
+      # @return [Hash] what the file and the snapshot carry
+      def to_json_hash = { pid:, host:, started_at: started_at.utc.iso8601, foreign: foreign? }
+    end
+
+    # The lock file's name in a plan folder, where releases before the
+    # runtime state directory wrote it. Read, never written.
+    Lock::FILENAME = "locked-by.pid"
+
+    # Seconds an unreadable lock is taken for one being written. Between the
+    # `O_EXCL` create and the write the file is empty; older than this, it is
+    # debris from a crash and is cleared like a dead pid's.
+    Lock::UNREADABLE_GRACE = 5
+
+    class Lock
+      # Another live run holds the plan's lock; the claim was not made.
+      class Held < OpenLanes::Error; end
+    end
+  end
+end

data/lib/openlanes/plans/mailbox.rb

@@ -36,8 +36,12 @@ module OpenLanes
 
       # One entry.
       #
+      # @!attribute [r] id
+      #   @return [String, nil] what a drained copy is recognised by; nil on
+      #     messages from before ids, and on every mailbox.md one
       # @!attribute [r] number
-      #   @return [Integer] 1-based, in order of writing
+      #   @return [Integer] 1-based, in order of writing, within one copy of
+      #     the plan's files
       # @!attribute [r] at
       #   @return [String] when, as {TIME_FORMAT} writes it
       # @!attribute [r] from
@@ -46,7 +50,18 @@ module OpenLanes
       #   @return [String] the agent it is for
       # @!attribute [r] body
       #   @return [String]
-      Message = Data.define(:number, :at, :from, :to, :body) do
+      # @!attribute [r] read
+      #   @return [String] "yes" once its recipient ran `mail ack`, "no" until
+      #     then. Set by the reader, never by delivery: a stream ack says the
+      #     bytes arrived, not that anybody read them.
+      Message = Data.define(:number, :at, :from, :to, :body, :read, :id) do
+        # A message written before acks existed carries no `read`, and reads
+        # as unread: nobody ever said they read it.
+        def initialize(number:, at:, from:, to:, body:, read: "no", id: nil) = super
+
+        # @return [Boolean]
+        def read? = read == "yes"
+
         # @return [String] as it stands in the file
         def to_s = "## #{number}#{SEPARATOR}#{at}#{SEPARATOR}#{from} → #{to}\n\n#{body.strip}\n"
       end
@@ -109,6 +124,100 @@ module OpenLanes
         from_state(state.message!(from:, to:, body: text, at:, after: legacy.map(&:number).max.to_i, command:))
       end
 
+      # Keep a message the harness drained from Redis, unless this plan
+      # already has it, here or in `mailbox.md`. A different message whose
+      # number is taken is kept under the next free number: see
+      # {PlanState#deliver!}.
+      #
+      # @param message [Hash] as {Live::Mail#drain} returns it
+      # @return [OpenLanes::Plans::Mailbox::Message, nil] as written, nil when
+      #   it was already here
+      # @raise [OpenLanes::Error] for one with no positive number, no body,
+      #   or a sender or recipient that is not one agent name
+      def deliver(message)
+        number = message["number"]
+        raise Error, "a delivered message needs a positive number" unless number.is_a?(Integer) && number.positive?
+        raise Error, "a message needs a body" if message["body"].to_s.strip.empty?
+        raise Error, "a message needs --from and --to, each one agent name" unless name?(message["from"]) && name?(message["to"])
+
+        old = legacy
+        return nil if old.any? { |m| same_as_legacy?(m, message) }
+
+        state.deliver!(message, reserved: old.map(&:number))&.then { from_state(it) }
+      end
+
+      # How many messages nobody has acked. The dashboard shows it, and
+      # nothing else reads it: an unacked message is a warning, never an input
+      # to a round (D3).
+      #
+      # @param recipient [String, nil] one reader, or everybody
+      # @return [Integer]
+      def unacked(recipient = nil)
+        messages.count { |m| !m.read? && (recipient.nil? || m.to == recipient) }
+      end
+
+      # The unacked messages, one entry per sender and recipient, so the
+      # dashboard can say who is waiting on whom rather than only how many.
+      #
+      # @return [Array<Hash{Symbol => String, Integer}>] `from`, `to`, `count`
+      #   and `since`, the oldest one's timestamp as written, oldest pair first
+      def unacked_pairs
+        messages.reject(&:read?).group_by { [it.from, it.to] }
+                .map { |(from, to), sent| { from:, to:, count: sent.size, since: sent.map(&:at).min.to_s } }
+                .sort_by { it[:since] }
+      end
+
+      # Whether +number+ is a `mailbox.md` message: history, counted as read,
+      # where an ack records nothing.
+      #
+      # @param number [Integer, String]
+      # @return [Boolean]
+      def legacy?(number)
+        number = Integer(number.to_s, exception: false)
+        legacy.any? { |m| m.number == number }
+      end
+
+      # Mark one message read. A `mailbox.md` number is returned as it
+      # stands, with nothing written: ask {#legacy?} before saying it was
+      # recorded.
+      #
+      # @param number [Integer]
+      # @param by [String] the reader, who must be its recipient
+      # @return [OpenLanes::Plans::Mailbox::Message] as it stands now
+      # @raise [OpenLanes::Error] for a number with no message, or a reader
+      #   it was not addressed to
+      def ack(number, by:)
+        number = Integer(number.to_s, exception: false)
+        raise Error, "a message number is a positive integer" unless number&.positive?
+
+        if (old = legacy.find { |m| m.number == number })
+          raise Error, "message ##{number} is for #{old.to}, not #{by}" unless old.to == by
+
+          return old
+        end
+
+        message = state.ack!(number, by:)
+        raise Error, "no message ##{number} in #{File.basename(dir)}" unless message
+
+        from_state(message)
+      end
+
+      # The whole exchange as Markdown, for a person after the round. Returns
+      # the text and writes nothing: the JSON is the record, and a rendered
+      # copy beside it would be a second one to drift.
+      #
+      # @return [String]
+      def render
+        title = "# Mailbox: #{File.basename(dir)}\n"
+        return "#{title}\nNo messages.\n" if messages.empty?
+
+        entries = messages.map do |m|
+          "## #{m.number}#{SEPARATOR}#{m.at}#{SEPARATOR}#{m.from} → #{m.to}#{SEPARATOR}" \
+            "#{m.read? ? "read" : "unread"}\n\n#{m.body.strip}\n"
+        end
+        ([title] + entries).join("\n")
+      end
+
       private
 
       # @return [OpenLanes::Plans::PlanState]
@@ -129,7 +238,32 @@ module OpenLanes
         rescue ArgumentError
           message["at"].to_s
         end
-        Message.new(number: message["number"], at:, from: message["from"], to: message["to"], body: message["body"])
+        Message.new(number: message["number"],
+          at:,
+          from: message["from"],
+          to: message["to"],
+          body: message["body"],
+          read: message["read"] == "yes" ? "yes" : "no",
+          id: message["id"])
+      end
+
+      # mailbox.md has no ids, so a drained copy of one of its messages is
+      # recognised by what it says.
+      #
+      # @param old [OpenLanes::Plans::Mailbox::Message]
+      # @param message [Hash]
+      # @return [Boolean]
+      def same_as_legacy?(old, message)
+        old.from == message["from"] && old.to == message["to"] && old.body.strip == message["body"].to_s.strip &&
+          time_of(old.at) == time_of(message["at"])
+      end
+
+      # @param text [String, nil]
+      # @return [Time, nil]
+      def time_of(text)
+        Time.parse(text.to_s)
+      rescue ArgumentError
+        nil
       end
 
       # @param value [Object]
@@ -149,7 +283,14 @@ module OpenLanes
         text.to_s.each_line(chomp: true) do |line|
           if (heading = HEADING.match(line))
             messages << current.with(body: body.join("\n").strip) if current
-            current = Message.new(number: heading[1].to_i, at: heading[2], from: heading[3], to: heading[4], body: "")
+            # mailbox.md predates acking, and nobody is left to ack it: it
+            # is history, so it counts as read rather than as a warning forever.
+            current = Message.new(number: heading[1].to_i,
+              at: heading[2],
+              from: heading[3],
+              to: heading[4],
+              body: "",
+              read: "yes")
             body = []
           elsif current
             body << line