openlanes 3.0.0 → 3.1.0

data/lib/openlanes/web/request_log.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require "rack/body_proxy"
+require "rack/utils"
+
+module OpenLanes
+  module Web
+    # One log entry per request, through {Log}, wrapped around the whole app
+    # so a request Rack::Protection or host authorization turns away is
+    # logged too.
+    #
+    # Not Sinatra's `logging` or Rack::CommonLogger: both write Apache-style
+    # text to `rack.errors`, which is the launcher's raw output file rather
+    # than the log the engine writes, and neither knows a page from a request.
+    #
+    # The event stream is the exception to "one entry per request". It stays
+    # open as long as the page does, so it is logged twice: when it opens, and
+    # when Puma closes its body, with how long it was open. Logged once, at
+    # the end, a page left open all day would not appear until evening.
+    #
+    # Static assets and `/healthz` are logged at debug: the launcher polls
+    # `/healthz` ten times a second while it waits, and a page load fetches a
+    # dozen assets, so at info they would bury the API calls.
+    class RequestLog
+      # A response of this type is a stream that outlives its request.
+      STREAM = "text/event-stream"
+
+      # Paths logged at info. Everything else is the page's own files.
+      API = %r{\A/api(/|\z)}
+
+      # @param app [#call]
+      # @param logger [#info, #debug, #error, nil] {Log.logger} when nil, read
+      #   per request so a {Log.setup} after this was built still applies
+      # @param clock [#call] monotonic seconds
+      def initialize(app, logger: nil, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
+        @app = app
+        @logger = logger
+        @clock = clock
+      end
+
+      # @param env [Hash] the Rack environment
+      # @return [Array] status, headers and body, the body wrapped for a stream
+      def call(env)
+        started = @clock.call
+        status, headers, body = @app.call(env)
+        return [status, headers, opened(env, status, started, body)] if stream?(headers)
+
+        write(level(env), env, "#{line(env)} #{status}", status:, started:)
+        [status, headers, body]
+      rescue StandardError => e
+        write(:error, env, "#{line(env)} raised #{e.class}", status: 500, started:, exception: e)
+        raise
+      end
+
+      private
+
+      # Logs the stream opening, and hands back a body that logs its close.
+      #
+      # @return [Rack::BodyProxy]
+      def opened(env, status, started, body)
+        write(:info, env, "#{line(env)} stream opened", status:)
+        Rack::BodyProxy.new(body) { write(:info, env, "#{line(env)} stream closed", status:, started:) }
+      end
+
+      # @param headers [Hash]
+      # @return [Boolean]
+      def stream?(headers)
+        type = headers["content-type"] || headers["Content-Type"]
+        type.to_s.start_with?(STREAM)
+      end
+
+      # @param env [Hash]
+      # @return [Symbol]
+      def level(env) = env["PATH_INFO"].to_s.match?(API) ? :info : :debug
+
+      # @param env [Hash]
+      # @return [String] e.g. "GET /api/board"
+      def line(env) = "#{env["REQUEST_METHOD"]} #{env["PATH_INFO"]}"
+
+      # The page's own id, from `?page=`, so the requests one browser tab
+      # made can be picked out of several tabs' worth.
+      #
+      # @param env [Hash]
+      # @return [String, nil]
+      def page(env)
+        id = Rack::Utils.parse_query(env["QUERY_STRING"].to_s)["page"]
+        id.is_a?(String) && !id.empty? ? id : nil
+      end
+
+      # @param level [Symbol]
+      # @param env [Hash]
+      # @param message [String]
+      # @param status [Integer]
+      # @param started [Float, nil] when given, the entry carries the duration since
+      # @param exception [Exception, nil]
+      # @return [void]
+      def write(level, env, message, status:, started: nil, exception: nil)
+        payload = { method: env["REQUEST_METHOD"], path: env["PATH_INFO"], status: status.to_i, page: page(env) }.compact
+        entry = { message:, payload:, exception: }.compact
+        entry[:duration] = ((@clock.call - started) * 1000).round(1) if started
+        (@logger || Log.logger).public_send(level, **entry)
+      rescue StandardError
+        # A log that cannot be written is no reason to fail the request.
+        nil
+      end
+    end
+  end
+end

data/lib/openlanes/web/server.rb

@@ -7,7 +7,8 @@ require "puma/server"
 module OpenLanes
   module Web
     # The app on Puma, multi-threaded because every open page holds one
-    # thread for its event stream, and bound to {HOST} only.
+    # thread for its event stream, and bound to {HOST} unless a {Listen}
+    # names other addresses.
     #
     #   server = Web.server(app:, context:, port: 11000).start
     #   server.url   # => "http://127.0.0.1:11000"
@@ -18,29 +19,31 @@ module OpenLanes
       MAX_THREADS = 64
 
       # Seconds Puma waits for open streams after {#stop}. They notice the
-      # context's `stopping` within a tick; this is the backstop.
+      # context's `stopping` on the `stop` nudge {#stop} publishes; this is
+      # the backstop, for a Redis that went away and took the nudge with it.
       SHUTDOWN_AFTER = 2
 
       # @param app [#call]
       # @param context [OpenLanes::Web::Context]
       # @param port [Integer] 0 for any free port
-      # @param host [String]
+      # @param listen [OpenLanes::Web::Listen] the addresses to listen on
       # @param log [IO] where Puma writes its errors
-      def initialize(app:, context:, port: DEFAULT_PORT, host: HOST, log: $stderr)
+      def initialize(app:, context:, port: DEFAULT_PORT, listen: Listen.parse(nil), log: $stderr)
         @app = app
         @context = context
         @port = Integer(port)
-        @host = host
+        @listen = listen
         @log = log
       end
 
-      # @return [String]
-      attr_reader :host
+      # @return [OpenLanes::Web::Listen]
+      attr_reader :listen
 
       # Listen, and serve on Puma's own threads.
       #
       # @return [self]
       # @raise [Errno::EADDRINUSE] when the port is taken
+      # @raise [Errno::EADDRNOTAVAIL] when an address is not this machine's
       def start
         @puma = Puma::Server.new(@app,
           nil,
@@ -48,16 +51,29 @@ module OpenLanes
           max_threads:          MAX_THREADS,
           force_shutdown_after: SHUTDOWN_AFTER,
           log_writer:           Puma::LogWriter.new(File.open(File::NULL, "w"), @log))
-        @puma.add_tcp_listener(@host, @port)
+        # Port 0 is resolved by the first listener, and the rest share it, so
+        # every address answers on the one port {#url} reports.
+        @listen.ips.each { @puma.add_tcp_listener(it, @puma.connected_ports.first || @port) }
         @thread = @puma.run
+        @context.chat&.start
         self
+      rescue SystemCallError
+        # A later address failing leaves the earlier ones bound; Puma is not
+        # running yet, so close its sockets here rather than leak them to
+        # whoever retries in this process.
+        @puma&.binder&.close
+        @puma = nil
+        raise
       end
 
       # @return [Integer] the port listened on, the real one when 0 was asked
       def port = @puma&.connected_ports&.first || @port
 
-      # @return [String]
-      def url = "http://#{@host}:#{port}"
+      # @return [String] where this machine's browser opens the page
+      def url = @listen.browse_url(port)
+
+      # @return [Array<String>] every address the page answers on
+      def urls = @listen.urls(port)
 
       # Block until the server stops.
       #
@@ -69,6 +85,11 @@ module OpenLanes
       # @return [void]
       def stop
         @context.stopping.make_true
+        # Wakes every stream blocked waiting for news, here and on any other
+        # server following this project, so ours see `stopping` now rather
+        # than at their next ping. The others look, find nothing, and sleep.
+        Live::News.publish(store: @context.store, keys: @context.keys, kind: "stop")
+        @context.chat&.stop
         @puma&.stop(true)
         @context.stores.close
       end

data/lib/openlanes/web/stores.rb

@@ -26,6 +26,17 @@ module OpenLanes
       # @return [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore]
       def current = @mutex.synchronize { @by_thread[Thread.current] ||= @factory.call }
 
+      # Drop and close this thread's store. A thread that is not one of
+      # Puma's pooled ones calls it when done, or the map would keep a
+      # connection for every such thread that ever ran. A shared store is
+      # dropped but not closed: the others still hold it.
+      #
+      # @return [void]
+      def release
+        store = @mutex.synchronize { @by_thread.delete(Thread.current) }
+        store&.close unless @mutex.synchronize { @by_thread.value?(store) }
+      end
+
       # @return [void]
       def close
         stores = @mutex.synchronize { @by_thread.values.uniq.tap { @by_thread.clear } }

data/lib/openlanes/web.rb

@@ -25,8 +25,8 @@ module OpenLanes
     # Where the server listens unless told otherwise (contract.md).
     DEFAULT_PORT = 11_000
 
-    # Loopback only: the page can stop agents, and nothing off this machine
-    # may reach it (W3).
+    # Loopback by default: the page can stop agents and has no login, so
+    # nothing off this machine reaches it unless `--ips` or `-I` says so (W3).
     HOST = "127.0.0.1"
 
     module_function
@@ -38,12 +38,27 @@ module OpenLanes
     # @param rest [Hash] {Context}'s other fields
     # @return [OpenLanes::Web::Context]
     def context(stores:, keys:, plans_dir:, url:, **rest)
+      rest[:chat] ||= chat(stores:, keys:, node: rest[:node])
       Context.new(stores:, keys:, plans_dir:, url:, csrf: SecureRandom.hex(32), **rest)
     end
 
+    # The chat between machines for one server (plan 006). Its machine is
+    # the node id in +keys+, so it answers for the same machine the board does.
+    #
+    # @param stores [OpenLanes::Web::Stores]
+    # @param keys [OpenLanes::Live::Keys]
+    # @param node [OpenLanes::Live::Node, nil]
+    # @return [OpenLanes::Live::Chat]
+    def chat(stores:, keys:, node: nil, **)
+      machine = keys.node.split(":").first
+      Live::Chat.new(stores:, keys:, machine:, label: node&.label || machine, **)
+    end
+
+    # The routes, with every request logged on the way out (see {RequestLog}).
+    #
     # @param context [OpenLanes::Web::Context]
-    # @return [Class] a Rack application
-    def app(context) = App.serving(context)
+    # @return [#call] a Rack application
+    def app(context) = RequestLog.new(App.serving(context))
 
     # @param app [#call]
     # @param context [OpenLanes::Web::Context]

data/lib/openlanes.rb

@@ -6,7 +6,10 @@ require "dry/cli"
 require "dry/cli/help"
 
 require "dry/inflector"
+# Every command workflow answers a Dry::Monads::Result (see OpenLanes::Refusal).
+require "dry/monads"
 require "fileutils"
+require "open3"
 require "shellwords"
 require "zeitwerk"
 loader = Zeitwerk::Loader.for_gem
@@ -21,13 +24,16 @@ loader.inflector.inflect(
   "ui"         => "UI"
 )
 loader.collapse("#{__dir__}/openlanes/support")
+loader.collapse("#{__dir__}/openlanes/cli/coverage")
 loader.collapse("#{__dir__}/openlanes/cli/create")
 loader.collapse("#{__dir__}/openlanes/cli/docs")
 loader.collapse("#{__dir__}/openlanes/cli/eval")
 loader.collapse("#{__dir__}/openlanes/cli/index")
 loader.collapse("#{__dir__}/openlanes/cli/init")
 loader.collapse("#{__dir__}/openlanes/cli/list_plans")
+loader.collapse("#{__dir__}/openlanes/cli/review")
 loader.collapse("#{__dir__}/openlanes/cli/run")
+loader.collapse("#{__dir__}/openlanes/cli/setup")
 loader.collapse("#{__dir__}/openlanes/cli/start")
 loader.collapse("#{__dir__}/openlanes/cli/states")
 loader.collapse("#{__dir__}/openlanes/cli/unblock")
@@ -40,9 +46,13 @@ loader.collapse("#{__dir__}/openlanes/cli/mail/subcommands")
 loader.collapse("#{__dir__}/openlanes/cli/state/subcommands")
 loader.collapse("#{__dir__}/openlanes/cli/resync/subcommands")
 loader.collapse("#{__dir__}/openlanes/cli/skills/subcommands")
+loader.collapse("#{__dir__}/openlanes/cli/api/subcommands")
+loader.ignore("#{__dir__}/openlanes/cli/api/api.rb")
+loader.collapse("#{__dir__}/openlanes/cli/app/subcommands")
 loader.ignore("#{__dir__}/openlanes/cli/linear/linear.rb")
 loader.ignore("#{__dir__}/openlanes/cli/mail/mail.rb")
 loader.ignore("#{__dir__}/openlanes/cli/state/state.rb")
+loader.ignore("#{__dir__}/openlanes/cli/app/app.rb")
 loader.ignore("#{__dir__}/openlanes/linear/mapping.rb")
 loader.setup
 
@@ -51,8 +61,8 @@ loader.setup
 # Three phases, each with a file that proves it happened, and a guarded state
 # machine that refuses to let a plan folder claim a phase it has not reached.
 #
-#   spec   ⚪️ New          spec.md
-#   plan   ⭐️ Ready        plan.md
+#   spec   ⚪ New          spec.md
+#   plan   ⭐ Ready        plan.md
 #   build  🟡 → ✅         pull-requests.md
 #
 # The conventions are not documented anywhere else by hand: the status
@@ -95,6 +105,8 @@ module OpenLanes
   def self.move_directory(source, target)
     return false if File.exist?(target)
 
+    # Into archive/ and friends, which the first move there creates.
+    FileUtils.mkdir_p(File.dirname(target))
     parent = File.dirname(source)
     tracked = system("git",
       "-C",
@@ -116,7 +128,8 @@ module OpenLanes
     true
   end
 
-  # Plan numbers visible in `.plans` on a git ref, without checking it out.
+  # Plan numbers visible in `.plans` on a git ref, without checking it out,
+  # the inactive subfolders included: an archived plan keeps its number.
   #
   # A pull request's branch is the only place that knows how far the sequence
   # had got when the work started, and branches are not all rebased onto the
@@ -127,14 +140,39 @@ module OpenLanes
   # @return [Array<OpenLanes::Plans::Ordinal>] possibly empty
   def self.plans_on_ref(root, ref)
     ["origin/#{ref}", ref].each do |candidate|
-      out = `git -C #{root.shellescape} ls-tree -d --name-only #{candidate.shellescape} #{PLANS_DIR}/ 2>/dev/null`
-      next if out.to_s.strip.empty?
-
-      return out.lines.filter_map { |line| Plans::Ordinal.from_dirname(File.basename(line.strip)) }
+      found = plans_at(root, candidate)
+      return found unless found.empty?
     end
     []
   end
 
+  # Plan numbers in `.plans` on one tree-ish, the inactive subfolders
+  # included. {plans_on_ref} guesses at a branch name; this reads exactly
+  # what it is given, which is what allocating a number across every branch
+  # needs: a full ref name tried as `origin/refs/heads/x` would cost a
+  # second `ls-tree` per branch for nothing.
+  #
+  # @param root [String] any directory inside the repository
+  # @param treeish [String] a ref, or a commit
+  # @param plans [String] the `.plans` directory, relative to +root+
+  # @return [Array<OpenLanes::Plans::Ordinal>] empty when the ref is unknown
+  #   or carries no plans
+  def self.plans_at(root, treeish, plans: PLANS_DIR)
+    dirs = [plans, *Plans::Tree::SUBFOLDERS.map { "#{plans}/#{it}" }].map { "#{it}/" }
+    out, status = Open3.capture2("git",
+      "-C",
+      root,
+      "ls-tree",
+      "-d",
+      "--name-only",
+      treeish,
+      *dirs,
+      err: File::NULL)
+    return [] unless status.success?
+
+    out.lines.filter_map { |line| Plans::Ordinal.from_dirname(File.basename(line.strip)) }
+  end
+
   # The agent that writes a specification for work that already shipped.
   # Named here rather than in the CLI so the definition file stays the single
   # source of truth about who does what.

data/openlanes.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   TEXT
   spec.homepage = "https://github.com/kigster/openlanes"
 
-  spec.required_ruby_version = ">= 3" # rubocop:disable Gemspec/RequiredRubyVersion -- the floor users get, not the Ruby the cops lint for
+  spec.required_ruby_version = ">= 4" # rubocop:disable Gemspec/RequiredRubyVersion -- the floor users get, not the Ruby the cops lint for
 
   spec.metadata = {
     "homepage_uri"          => spec.homepage,
@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
     # `openlanes web --demo` replays.
     # Source maps stay out: they double the page's size for nobody who installs it.
     globbed = Dir.glob(["*.gemspec", "lib/**/*.rb", "agents/*.md", "schemas/*.json", "evals/cases/**/*", "skills/**/*",
-                        "exe/*", "bin/*", "web/dist/**/*", "web/src/api/fixtures/board.json"])
+                        "exe/*", "web/dist/**/*", "web/src/api/fixtures/board.json", "Brewfile"])
     globbed.select { |f| File.file?(f) && !f.end_with?(".map") }
   end
 
@@ -49,10 +49,10 @@ Gem::Specification.new do |spec|
   end
 
   spec.bindir = "exe"
-  # `depot` and `lanes` are symlinks to `openlanes` in the checkout; named here
+  # `ol` and `lanes` are symlinks to `openlanes` in the checkout; named here
   # so that installing the gem writes all three, rather than only the one git
   # recorded.
-  spec.executables = %w[openlanes depot lanes]
+  spec.executables = %w[openlanes ol lanes]
   spec.require_paths = ["lib"]
 
   spec.add_dependency "aasm"
@@ -66,6 +66,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "dry-inflector"
   spec.add_dependency "dry-monads"
   spec.add_dependency "fuzzy-string-match"
+  spec.add_dependency "inquirex-llm", "~> 0.12"
   # Opens the control room in the browser when `openlanes run` starts the API server.
   spec.add_dependency "launchy", "~> 3.1"
   spec.add_dependency "parallel"
@@ -76,7 +77,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "rackup", "~> 2.1"
   # The live run (plan 004) lives in Redis: board and command streams, plan claims.
   spec.add_dependency "redis-client", "~> 0.31"
+  spec.add_dependency "semantic_logger", "~> 5.1"
   spec.add_dependency "sinatra", "~> 4.2"
+  spec.add_dependency "agent-lock"
   spec.add_dependency "tty-box"
   spec.add_dependency "tty-command"
   spec.add_dependency "tty-cursor"
@@ -84,4 +87,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency "tty-screen"
   spec.add_dependency "unicode-display_width"
   spec.add_dependency "zeitwerk"
+  spec.add_dependency "newrelic_rpm"
+  spec.add_dependency "newrelic-infinite_tracing"
 end

data/schemas/journal-event.schema.json

@@ -0,0 +1,72 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/kigster/openlanes/schemas/journal-event.schema.json",
+  "title": "openlanes engine journal event",
+  "description": "One line of a node's NDJSON progress journal, written by OpenLanes::Engine::EventJournal under Runtime.journal_dir. Append-only, fsync'd per event.",
+  "type": "object",
+  "required": ["v", "seq", "at", "node", "run_id", "kind"],
+  "properties": {
+    "v": { "const": 1 },
+    "seq": { "type": "integer", "minimum": 1 },
+    "at": { "type": "string", "format": "date-time" },
+    "node": { "type": "string", "minLength": 1 },
+    "run_id": { "type": "string", "minLength": 1 },
+    "kind": {
+      "enum": [
+        "run_started", "run_ended", "plan_assigned", "agent_started", "agent_session",
+        "agent_offset", "agent_ended", "transition", "extension", "question"
+      ]
+    },
+    "plan": { "type": "string", "pattern": "^\\d{3}\\.\\d{2}$" },
+    "agent": { "type": "string", "minLength": 1 },
+    "round": { "type": "integer", "minimum": 0 },
+    "pid": { "type": "integer", "minimum": 1 },
+    "started_at": { "type": ["string", "null"], "format": "date-time" },
+    "session_id": { "type": ["string", "null"] },
+    "output_file": { "type": ["string", "null"] },
+    "offset": { "type": "integer", "minimum": 0 },
+    "exit": { "type": ["integer", "null"] },
+    "from": { "type": "string" },
+    "to": { "type": "string" },
+    "event": { "type": "string" },
+    "seconds": { "type": "number", "minimum": 0 },
+    "note": { "type": "string" },
+    "reason": { "type": "string" },
+    "jobs": { "type": "integer", "minimum": 1 },
+    "isolation": { "type": "string" }
+  },
+  "allOf": [
+    {
+      "if": { "properties": { "kind": { "const": "plan_assigned" } } },
+      "then": { "required": ["plan", "agent"] }
+    },
+    {
+      "if": { "properties": { "kind": { "const": "agent_started" } } },
+      "then": { "required": ["plan", "agent", "pid", "started_at", "session_id", "output_file"] }
+    },
+    {
+      "if": { "properties": { "kind": { "const": "agent_session" } } },
+      "then": { "required": ["plan", "agent", "session_id"], "properties": { "session_id": { "type": "string" } } }
+    },
+    {
+      "if": { "properties": { "kind": { "const": "agent_offset" } } },
+      "then": { "required": ["plan", "agent", "offset"] }
+    },
+    {
+      "if": { "properties": { "kind": { "const": "agent_ended" } } },
+      "then": { "required": ["plan", "agent", "exit"] }
+    },
+    {
+      "if": { "properties": { "kind": { "const": "transition" } } },
+      "then": { "required": ["plan", "from", "to"] }
+    },
+    {
+      "if": { "properties": { "kind": { "const": "extension" } } },
+      "then": { "required": ["plan", "agent", "seconds"] }
+    },
+    {
+      "if": { "properties": { "kind": { "const": "question" } } },
+      "then": { "required": ["plan", "agent", "note"] }
+    }
+  ]
+}