openlanes 3.0.0 → 3.1.0

data/lib/openlanes/execution/control.rb

@@ -37,6 +37,14 @@ module OpenLanes
       # compacts its session from outside (see {Executor::Handle#compact!}).
       WORDS = { "hold" => HOLD, "wrap_up" => WRAP_UP, "stop" => STOP }.freeze
 
+      # Redis stopped answering (W32): every agent stops at its next safe
+      # point and waits until the file says anything else. Its clock stands
+      # still meanwhile, so the wait costs it no time.
+      PAUSE = "PAUSE"
+
+      # Redis is back: carry on from where the PAUSE found you.
+      RESUME = "RESUME"
+
       # Seconds between {#quit!} and the harness terminating whatever is still
       # running. Long enough to write files and a handoff note; short enough
       # that q means quit rather than "quit eventually".
@@ -46,6 +54,7 @@ module OpenLanes
       @files = []
       @quit = false
       @interrupted = false
+      @paused = false
       @deadline = nil
 
       class << self
@@ -107,6 +116,9 @@ module OpenLanes
         def quit!
           stop!
           @mutex.synchronize {
+            # Quitting overrides a pause: the clocks run again, so an agent
+            # that does not stop is still stopped.
+            @paused = false
             @quit = true
             @deadline ||= UI.monotonic + GRACE
           }
@@ -127,6 +139,25 @@ module OpenLanes
           }
         end
 
+        # Every running agent waits where it is (W32).
+        #
+        # @return [void]
+        def pause!
+          @mutex.synchronize { @paused = true }
+          broadcast(PAUSE)
+        end
+
+        # Every paused agent carries on.
+        #
+        # @return [void]
+        def resume!
+          @mutex.synchronize { @paused = false }
+          broadcast(RESUME)
+        end
+
+        # @return [Boolean] whether agents are paused; their clocks stand still
+        def paused? = @mutex.synchronize { @paused }
+
         # @return [Boolean] whether `q` has been pressed
         def quit? = @mutex.synchronize { @quit }
 
@@ -172,6 +203,7 @@ module OpenLanes
             @files.each { |f| FileUtils.rm_f(f) }
             @files.clear
             @quit = false
+            @paused = false
             @interrupted = false
             @deadline = nil
           }

data/lib/openlanes/execution/executor.rb

@@ -15,9 +15,10 @@ module OpenLanes
     #
     #   BEFORE — the agent is told, and `--disallowedTools` withholds the tools
     #            that would let it push.
-    #   AFTER  — the harness checks that HEAD did not move and no new remote ref
-    #            appeared. A prompt is a request; a check is a guarantee, and only
-    #            one of them survives a model deciding it knows better.
+    #   AFTER  — {Boundary} checks that HEAD did not move and that no
+    #            remote-tracking ref was moved by a push. A prompt is a request;
+    #            a check is a guarantee, and only one of them survives a model
+    #            deciding it knows better.
     class Executor
       # What one invocation did, and what it spent doing it.
       #
@@ -124,12 +125,30 @@ module OpenLanes
         # @return [String, nil] the CLI's session id, once its stream named it
         attr_accessor :session
 
+        # @return [String, nil] the raw NDJSON trace the agent's stream is
+        #   copied into, so the engine journal can name it and its offset
+        attr_accessor :trace
+
         # @return [OpenLanes::Execution::Executor::Compaction, nil] the latest
         #   compaction asked for
         def compaction = @mutex.synchronize { @compaction }
 
         # @return [OpenLanes::Execution::Clock, nil]
-        attr_accessor :clock
+        attr_reader :clock
+
+        # The clock, once {Executor#call} has started it. Seconds granted
+        # before then (a round given more time, E1) go onto it now rather
+        # than being dropped for want of a clock.
+        #
+        # @param clock [OpenLanes::Execution::Clock, nil]
+        # @return [void]
+        def clock=(clock)
+          @mutex.synchronize do
+            @clock = clock
+            clock&.extend!(@pending) if @pending&.positive?
+            @pending = nil
+          end
+        end
 
         # @return [String, nil]
         attr_reader :control
@@ -184,9 +203,18 @@ module OpenLanes
           child&.terminate
         end
 
+        # More time on the clock, or on the clock to come.
+        #
         # @param seconds [Integer]
-        # @return [void]
-        def extend!(seconds) = clock&.extend!(seconds)
+        # @return [Boolean] false when the clock had already stopped the agent
+        def extend!(seconds)
+          @mutex.synchronize do
+            next @clock.extend!(seconds) if @clock
+
+            @pending = @pending.to_i + seconds
+            true
+          end
+        end
 
         # Ask for the session to be compacted (W28): INTERRUPT, so the agent
         # leaves a resume note and ends its turn, and SIGKILL after +grace+
@@ -290,7 +318,10 @@ module OpenLanes
         def tools = @all.sum(&:tools)
 
         # What the sessions already over spent, added to the running one's
-        # progress so the meter carries on rather than starting again.
+        # progress so the meter carries on rather than starting again. Their
+        # journal lines go first, for the same reason: the dispatcher counts
+        # its place in the list, and a list that started again at the
+        # continuation would have it skip the continuation's first entries.
         #
         # @param progress [OpenLanes::Execution::Transcript::Progress]
         # @return [OpenLanes::Execution::Transcript::Progress]
@@ -298,7 +329,8 @@ module OpenLanes
           before = @all[0...-1]
           progress.with(up: progress.up + before.sum(&:up),
             down: progress.down + before.sum(&:down),
-            subagents: progress.subagents + before.sum(&:spawned))
+            subagents: progress.subagents + before.sum(&:spawned),
+            journal: (before.flat_map { it.respond_to?(:journal) ? it.journal : [] } + progress.journal).freeze)
         end
       end
 
@@ -321,8 +353,13 @@ module OpenLanes
       # as a regular expression that nothing referenced — a guard in the shape of
       # a constant, enforcing nothing — which is exactly how `gh pr review` came
       # to be reachable by an agent nobody had granted it to.
+      #
+      # `git rebase` is here because the harness owns the rebase onto the
+      # default branch ({Vcs::Refresh}): an agent resolving a stopped one only
+      # edits and stages, and a `--continue`, `--skip` or `--abort` of its own
+      # would move HEAD inside the window the after-check guards.
       FORBIDDEN_COMMANDS = [
-        "git push", "git commit",
+        "git push", "git commit", "git rebase",
         "gh pr create", "gh pr edit", "gh pr merge",
         "gh pr review", "gh pr comment",
         "gh release create"
@@ -335,7 +372,11 @@ module OpenLanes
       # does not: an approval is reversible, visible, and attributable to the
       # identity that made it. That difference is the whole line between
       # `hansolo-reviewer` approving and `hansolo-reviewer` merging.
-      UNGRANTABLE = ["git push", "gh pr merge"].freeze
+      #
+      # A rebase is the harness's alone for the same reason a merge is: it
+      # rewrites the branch a pull request is built from, and the harness
+      # pushes that rewrite with a lease only it can check.
+      UNGRANTABLE = ["git push", "gh pr merge", "git rebase"].freeze
 
       # How much of what the agent said survives into a one-line report.
       REASON_LIMIT = 300
@@ -376,6 +417,11 @@ module OpenLanes
       # and nothing here can tell the two apart.
       CREDENTIAL_VARS = %w[ANTHROPIC_API_KEY ANTHROPIC_AUTH_TOKEN].freeze
 
+      # What `claude -p --resume <id>` prints for an id it has no session for,
+      # in the directory it was run from (2.1.x): this line on stderr, then a
+      # `result` event with `is_error`, zero turns and zero tokens, exit 1.
+      SESSION_MISSING = /No conversation found with session ID/
+
       # @param env [Hash]
       # @return [Array<String>] credential variables currently set
       def self.foreign_credentials(env = ENV)
@@ -476,9 +522,15 @@ module OpenLanes
       # @param handle [OpenLanes::Execution::Executor::Handle, nil] filled in for the caller
       # @param partners [Array<OpenLanes::Agents::Agent>] the other agents working this
       #   plan right now, named in the prompt beside the plan's mailbox
+      # @param rebase [OpenLanes::Vcs::Refresh::Outcome, nil] a rebase stopped
+      #   on conflicts: the invocation resolves it and does no plan work
+      # @param resume [String, nil] the previous round's session id (M1), to
+      #   carry on with this round's prompt; ignored by a CLI that cannot
+      #   resume, and dropped for a fresh start when the CLI cannot find it
       # @return [OpenLanes::Execution::Executor::Result]
       # @yieldparam progress [OpenLanes::Execution::Transcript::Progress]
-      def call(agent, subject, root: @root, round: 1, successor: nil, handle: nil, partners: [], &on_progress)
+      def call(agent, subject, root: @root, round: 1, successor: nil, handle: nil, partners: [], rebase: nil,
+        resume: nil, &on_progress)
         started = UI.monotonic
         if @dry_run
           return Result.new(ok: true,
@@ -491,12 +543,14 @@ module OpenLanes
         end
 
         handle ||= Handle.new
-        before = head(root)
+        before = Boundary.snapshot(root)
         trace = trace_path(agent, subject)
+        handle.trace = trace
         profile = profile_for(agent, subject)
         control = Control.register(@trace_dir, "#{subject.feature.ordinal}-#{agent.name}")
         handle.control = control
-        argv = invocation(agent, subject, root:, control:, round:, successor:, partners:)
+        resume = nil unless profile.adapter.resumes?
+        argv = invocation(agent, subject, root:, control:, round:, successor:, partners:, rebase:, resume:)
         sessions = Sessions.new
         child = launch(argv, root:, profile:, trace:, handle:, sessions:, &on_progress)
         clock = Clock.new(seconds: timeout_for(agent),
@@ -507,6 +561,15 @@ module OpenLanes
 
         begin
           status = drain(child, sessions, handle)
+          if resume && !handle.killed && (lost = lost_session(status, sessions.current))
+            UI.log("session #{resume} #{lost}, starting #{agent.name} fresh",
+              level: :warn,
+              plan:  subject.feature.ordinal.to_s,
+              agent: agent.name)
+            handle.session = nil
+            argv = invocation(agent, subject, root:, control:, round:, successor:, partners:, rebase:)
+            status = drain(launch(argv, root:, profile:, trace:, handle:, sessions:, &on_progress), sessions, handle)
+          end
           while (compaction = handle.take_compaction)
             resumed = settling(compaction, agent) do
               carry_on(compaction, agent, subject, root:, profile:, trace:, handle:, sessions:, &on_progress)
@@ -541,7 +604,7 @@ module OpenLanes
             pid: handle.pid)
         end
 
-        violation = boundary_violation(before, root)
+        violation = before.violation(root)
         return failure(sessions, started, violation, pid: handle.pid) if violation
 
         spent(sessions,
@@ -602,12 +665,15 @@ module OpenLanes
       # @param round [Integer] which attempt on this plan this is
       # @param successor [String, nil] who the agent names on its `next:` line
       # @param partners [Array<OpenLanes::Agents::Agent>]
+      # @param rebase [OpenLanes::Vcs::Refresh::Outcome, nil]
+      # @param resume [String, nil] a session id to carry on
       # @return [Array<String>]
-      def invocation(agent, subject, root: @root, control: nil, round: 1, successor: nil, partners: [])
+      def invocation(agent, subject, root: @root, control: nil, round: 1, successor: nil, partners: [], rebase: nil,
+        resume: nil)
         profile = profile_for(agent, subject)
         profile.adapter.argv(
           Adapters::Invocation.new(
-            prompt:          prompt_for(agent, subject, root, control:, round:, successor:, partners:, profile:),
+            prompt:          prompt_for(agent, subject, root, control:, round:, successor:, partners:, profile:, rebase:),
             root:,
             model:           profile.model,
             effort:          profile.effort,
@@ -615,7 +681,8 @@ module OpenLanes
             denied_tools:    denied_for(agent),
             denied_commands: denied_commands(agent),
             network:         agent.network,
-            lean:            @lean
+            lean:            @lean,
+            resume:
           )
         )
       end
@@ -659,6 +726,33 @@ module OpenLanes
         child
       end
 
+      # Whether a resumed invocation failed because its session is gone,
+      # rather than at anything it did, and what to say about it. The message
+      # is the sure sign; a failure that spent nothing and called no tool is
+      # taken as the same thing, because a CLI that rewords the message must
+      # still not cost the round. Starting such a failure again fresh costs
+      # one more start-up at worst: an invocation that spent nothing has
+      # nothing to lose.
+      #
+      # That second case also catches an overload, an auth error or a network
+      # drop at start-up, which the fresh start will usually hit again. The
+      # line the CLI printed goes into the log beside the retry, so an outage
+      # reads as an outage rather than as a lost session.
+      #
+      # @param status [Process::Status]
+      # @param transcript [OpenLanes::Execution::Transcript]
+      # @return [String, nil] why the session counts as lost, nil when it does not
+      def lost_session(status, transcript)
+        return nil if status.success? && !transcript.failed?
+
+        said = [transcript.error, *transcript.plain].map(&:to_s).reject(&:empty?)
+        return "not found" if said.any? { SESSION_MISSING.match?(it) }
+        return nil unless transcript.up.zero? && transcript.down.zero? && transcript.tools.zero?
+
+        detail = said.empty? ? "exit #{status.exitstatus}" : said.last.lines.first.strip[0, 200]
+        "failed before spending anything (#{detail})"
+      end
+
       # @return [Process::Status]
       def drain(child, sessions, handle)
         child.each_chunk do |out|
@@ -794,10 +888,12 @@ module OpenLanes
       # @param round [Integer]
       # @param successor [String, nil]
       # @param partners [Array<OpenLanes::Agents::Agent>]
+      # @param rebase [OpenLanes::Vcs::Refresh::Outcome, nil]
       # @return [String]
       def prompt_for(agent, subject, root = @root, control: nil, round: 1, successor: nil, partners: [],
-        profile: profile_for(agent, subject))
+        profile: profile_for(agent, subject), rebase: nil)
         Prompt.new(agent:,
+          rebase:,
           subject:,
           root:,
           profile:,
@@ -827,23 +923,6 @@ module OpenLanes
           handle.kill!(grace: 0, reason: :quit)
         end
       end
-
-      # @return [String, nil] current commit, or nil outside a repository
-      def head(root = @root)
-        out = `git -C #{root.shellescape} rev-parse HEAD 2>/dev/null`.strip
-        out.empty? ? nil : out
-      end
-
-      # @param before [String, nil]
-      # @return [String, nil] what boundary was crossed, or nil
-      def boundary_violation(before, root = @root)
-        return nil if before.nil?
-
-        after = head(root)
-        return "agent committed (HEAD moved #{before[0, 7]} → #{after[0, 7]}) — the boundary is docs plus code, no commits" if after != before
-
-        nil
-      end
     end
   end
 end

data/lib/openlanes/execution/process_start.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "etc"
+require "open3"
+require "time"
+
+module OpenLanes
+  module Execution
+    # When a process started, so a recovering engine can tell its own agent
+    # from a stranger that inherited the pid. Ruby's stdlib has no call for
+    # it: Linux answers through `/proc/<pid>/stat`, macOS through
+    # `ps -o lstart=`. The engine runs this, never an agent, so the autonomy
+    # boundary's withheld commands do not apply.
+    module ProcessStart
+      module_function
+
+      # @param pid [Integer]
+      # @param proc_root [String] where `/proc` is, a fixture in specs
+      # @param runner [#capture2] runs `ps`, injected in specs
+      # @return [Time, nil] the start time, or nil where it cannot be read
+      def of(pid, proc_root: "/proc", runner: Open3)
+        return nil unless pid.is_a?(Integer) && pid.positive?
+
+        from_proc(pid, proc_root) || from_ps(pid, runner)
+      end
+
+      # Field 22 of `stat` is the start in clock ticks after boot. The command
+      # name (field 2) may hold spaces and parentheses, so fields are counted
+      # from the last `)`.
+      #
+      # @api private
+      def from_proc(pid, proc_root)
+        stat = File.join(proc_root, pid.to_s, "stat")
+        uptime = File.join(proc_root, "uptime")
+        return nil unless File.file?(stat) && File.file?(uptime)
+
+        ticks = File.read(stat).split(")").last.split[19].to_i
+        booted = Time.now - File.read(uptime).split.first.to_f
+        booted + (ticks / clock_ticks.to_f)
+      rescue SystemCallError, NoMethodError
+        nil
+      end
+
+      # @api private
+      def from_ps(pid, runner)
+        out, status = runner.capture2("ps", "-o", "lstart=", "-p", pid.to_s)
+        return nil unless status.success?
+
+        text = out.strip
+        text.empty? ? nil : Time.parse(text)
+      rescue SystemCallError, ArgumentError
+        nil
+      end
+
+      # @api private
+      def clock_ticks
+        Etc.sysconf(Etc::SC_CLK_TCK)
+      rescue StandardError
+        100
+      end
+    end
+  end
+end