openlanes 3.0.1 → 3.1.0

data/lib/openlanes/web/launcher.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "net/http"
+require "socket"
 require "tmpdir"
 
 module OpenLanes
@@ -26,19 +27,35 @@ module OpenLanes
       # Seconds {#stop} gives the server after TERM before KILL.
       STOP_WITHIN = 5.0
 
+      # Ports {#start} tries, from the one asked for, before giving up.
+      PORTS_TRIED = 10
+
       # @param root [String] the checkout the run works in
       # @param plans_dir [String]
       # @param port [Integer]
       # @param redis_url [String]
-      # @param log [String] where the server's output goes
+      # @param log [String] where the server's raw STDOUT and STDERR go: a
+      #   crash before its logger is up, and Puma's own complaints
+      # @param logging [OpenLanes::Log::Settings, nil] where the server logs
+      #   its requests; the engine's own, so both processes write one file
       # @param ready_within [Float]
       # @param probe [Proc, nil] answers whether the server is up; `/healthz` by default
-      def initialize(root:, plans_dir:, port:, redis_url:, log: default_log(root), ready_within: READY_WITHIN, probe: nil)
+      # @param listen [OpenLanes::Web::Listen] the addresses the server listens on
+      # @param credentials [OpenLanes::Web::AdminAuth::Credentials, nil] its
+      #   login for requests from off this machine, handed over in {#env}
+      # @param spawner [#spawn] what starts the server process
+      def initialize(root:, plans_dir:, port:, redis_url:, log: default_log(root), logging: Log.settings,
+        ready_within: READY_WITHIN, probe: nil, listen: Listen.parse(nil), credentials: nil,
+        spawner: Spawner::System)
         @root = root
+        @spawner = spawner
         @plans_dir = plans_dir
         @port = Integer(port)
         @redis_url = redis_url
+        @listen = listen
+        @credentials = credentials
         @log = log
+        @logging = logging
         @ready_within = ready_within
         @probe = probe || method(:healthy?)
       end
@@ -49,22 +66,46 @@ module OpenLanes
       # @return [Integer, nil]
       attr_reader :pid
 
+      # @return [Integer] the port asked for, or the free one {#start} moved to
+      attr_reader :port
+
+      # Loopback, where the login is never asked. Every server it launches
+      # answers there: on loopback by default, and on a wildcard under -I.
+      #
       # @return [String]
       def url = "http://#{HOST}:#{@port}"
 
-      # @return [Array<String>] what is spawned; the Redis URL is in {#env}, never here
+      # @return [Array<String>] what is spawned; the Redis URL and the
+      #   credentials are in {#env}, never here
       def argv
-        [EXE, "web", "--port", @port.to_s, "--no-open", "--dir", @plans_dir]
+        ips = @listen.public? ? ["--ips", @listen.ips.join(",")] : []
+        [EXE, "web", "--port", @port.to_s, "--no-open", "--dir", @plans_dir, *ips]
       end
 
+      # The Redis URL, the login, and where to log. A server left to find its
+      # own log would resolve `log/development.log` from its own root and
+      # defaults, and a `--log` typed on `run` would move the engine's lines
+      # but not the server's.
+      #
       # @return [Hash{String => String}] added to the server's environment
-      def env = { REDIS_URL_ENV => @redis_url }
+      def env
+        login = @credentials ? { AdminAuth::ENV_NAME => "#{@credentials.user}:#{@credentials.password}" } : {}
+        { REDIS_URL_ENV => @redis_url, **login, **(@logging ? @logging.env : {}) }
+      end
 
-      # Spawn the server and wait for it to answer.
+      # Spawn the server on the first free port from the one asked for, and
+      # wait for it to answer. A taken port is skipped before spawning, never
+      # probed: a leftover `openlanes web` for another project answers
+      # `/healthz` there, and the run once declared that server its own and
+      # opened the browser on someone else's board.
       #
       # @return [self]
       def start
-        @pid = Process.spawn(env, *argv, chdir: @root, pgroup: true, in: File::NULL, %i[out err] => [@log, "a"])
+        free = free_port
+        return taken unless free
+
+        @port = free
+        @pid = @spawner.spawn(env, *argv, chdir: @root, pgroup: true, in: File::NULL, %i[out err] => [@log, "a"])
         @ready = wait_until_ready
         self
       end
@@ -88,6 +129,27 @@ module OpenLanes
 
       private
 
+      # @return [Integer, nil]
+      def free_port = (@port...(@port + PORTS_TRIED)).find { free?(it) }
+
+      # @param port [Integer]
+      # @return [Boolean] whether nothing listens there
+      def free?(port)
+        TCPServer.new(HOST, port).close
+        true
+      rescue SystemCallError
+        false
+      end
+
+      # Every port tried is taken: said in the log {#start}'s caller points at.
+      #
+      # @return [self]
+      def taken
+        File.write(@log, "Ports #{@port}-#{@port + PORTS_TRIED - 1} are all taken. Pass --port to use another.\n", mode: "a")
+        @ready = false
+        self
+      end
+
       # @return [Boolean]
       def wait_until_ready
         deadline = monotonic + @ready_within

data/lib/openlanes/web/listen.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require "ipaddr"
+require "socket"
+
+module OpenLanes
+  module Web
+    # The addresses one server listens on, and what follows from them: the
+    # Host headers it answers, the URLs it prints, the one a browser opens.
+    #
+    # Loopback, {HOST}, unless told otherwise (W3). The page can stop agents
+    # and answer their questions, and has no login, so listening beyond this
+    # machine is something a person asks for by name, and is warned about.
+    #
+    #   listen = Listen.parse("0.0.0.0")
+    #   listen.public?             # => true
+    #   listen.urls(11_000)        # => ["http://192.168.1.20:11000", ...]
+    #   listen.browse_url(11_000)  # => "http://127.0.0.1:11000"
+    Listen = Data.define(:ips) do
+      # @return [Boolean] whether anything off this machine can reach it
+      def public? = ips.any? { !IPAddr.new(it).loopback? }
+
+      # @return [Array<String>] the addresses past loopback, as given
+      def public_ips = ips.reject { IPAddr.new(it).loopback? }
+
+      # Where this machine's browser opens the page: loopback, which every
+      # wildcard listener also answers on, else the first address given.
+      #
+      # @param port [Integer]
+      # @return [String]
+      def browse_url(port)
+        ip = ips.find { Listen.wildcard?(it) || IPAddr.new(it).loopback? } || ips.first
+        Listen.url(if Listen.wildcard?(ip)
+                     IPAddr.new(ip).ipv6? ? "::1" : HOST
+                   else
+                     ip
+                   end,
+          port)
+      end
+
+      # Every URL the page answers on, a wildcard spelled out as this
+      # machine's own addresses, since `http://0.0.0.0` opens nothing.
+      #
+      # @param port [Integer]
+      # @param addresses [Array<String>] this machine's interface addresses
+      # @return [Array<String>]
+      def urls(port, addresses: Listen.addresses) = reached(addresses).map { Listen.url(it, port) }
+
+      # The Host headers answered beside loopback, which {App} always
+      # permits: each address listened on, a wildcard's interfaces, and this
+      # machine's name. Anything else is refused, so a page on another site
+      # still cannot reach the server through DNS rebinding.
+      #
+      # @param addresses [Array<String>]
+      # @param hostname [String]
+      # @return [Array<String, IPAddr>]
+      def permitted_hosts(addresses: Listen.addresses, hostname: Socket.gethostname)
+        return [] unless public?
+
+        names = [hostname, hostname.split(".").first].uniq
+        names + reached(addresses).map { IPAddr.new(it) }
+      end
+
+      private
+
+      # @param addresses [Array<String>]
+      # @return [Array<String>] the addresses listened on, each wildcard
+      #   replaced by the interface addresses it answers on
+      def reached(addresses) = ips.flat_map { Listen.wildcard?(it) ? Listen.covered(it, addresses) : [it] }.uniq
+    end
+
+    class Listen
+      # Raised for an address that is not one.
+      class Invalid < OpenLanes::Error; end
+
+      # What `-I` stands for.
+      EVERYWHERE = "0.0.0.0"
+
+      class << self
+        # An address a wildcard given beside it already covers is dropped:
+        # Linux refuses to bind `127.0.0.1` on a port `0.0.0.0` holds, even
+        # with SO_REUSEADDR, and the EADDRINUSE read as "port taken".
+        #
+        # @param text [String, nil] comma-separated addresses; blank for {HOST}
+        # @return [Listen]
+        # @raise [Invalid] naming the address that is not one
+        def parse(text)
+          ips = text.to_s.split(",").map(&:strip).reject(&:empty?)
+          ips = [HOST] if ips.empty?
+          ips.each do |ip|
+            IPAddr.new(ip)
+          rescue IPAddr::Error
+            raise Invalid, "#{ip.inspect} is not an IP address"
+          end
+          ips = ips.uniq
+          wildcards = ips.select { wildcard?(it) }
+          new(ips: ips.reject { |ip| wildcards.any? { it != ip && covered(it, [ip]).any? } })
+        end
+
+        # @param ip [String]
+        # @return [Boolean] whether it means every interface
+        def wildcard?(ip) = IPAddr.new(ip).to_i.zero?
+
+        # @param ip [String]
+        # @param port [Integer]
+        # @return [String]
+        def url(ip, port) = "http://#{IPAddr.new(ip).ipv6? ? "[#{ip}]" : ip}:#{port}"
+
+        # The interface addresses a wildcard answers on. `0.0.0.0` is IPv4
+        # only; `::` is both, because Ruby's TCPServer leaves IPV6_V6ONLY
+        # off, so Linux and macOS hand it IPv4 clients too. Treating `::` as
+        # IPv6 alone refused an IPv4 LAN client's Host header with a 403.
+        #
+        # @param wildcard [String] `0.0.0.0` or `::`
+        # @param addresses [Array<String>]
+        # @return [Array<String>]
+        def covered(wildcard, addresses)
+          return addresses if IPAddr.new(wildcard).ipv6?
+
+          addresses.select { IPAddr.new(it).ipv4? }
+        end
+
+        # This machine's interface addresses, loopback first, link-local
+        # IPv6 left out: it needs a zone to be dialed, which no URL carries.
+        #
+        # @return [Array<String>]
+        def addresses
+          Socket.ip_address_list.reject(&:ipv6_linklocal?).map(&:ip_address).sort_by { IPAddr.new(it).loopback? ? 0 : 1 }
+        end
+      end
+    end
+  end
+end

data/lib/openlanes/web/plan_files.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "securerandom"
+
+module OpenLanes
+  module Web
+    # The plan files the page may read, and the two it may write (plan 017).
+    #
+    # A name is looked up in {READABLE} and never joined onto a path as typed,
+    # which is what makes `../` meaningless here rather than merely refused.
+    # This knows nothing of HTTP or of cars: {App} maps each refusal to a
+    # status and checks the board before it calls {#write}.
+    class PlanFiles
+      # What the drawer lists and the preview reads, in the order listed.
+      READABLE = %w[spec.md plan.md contract.md blocked.md research.md design.md].freeze
+
+      # What the popup may save. `blocked.md` is appended by the `answer`
+      # command, and the build documents belong to the agents that own them.
+      EDITABLE = %w[spec.md plan.md].freeze
+
+      # Not a readable name, not a plan, or not a file in that plan.
+      class NotFound < OpenLanes::Error; end
+
+      # A readable name the page may not write.
+      class ReadOnly < OpenLanes::Error; end
+
+      # The file changed since the page read it.
+      class Stale < OpenLanes::Error; end
+
+      # The body's frontmatter block is not the one on disk. Frontmatter
+      # routes agents (`lane`, `frontend`, `depth`), so a crafted save must
+      # not reach it even when the page would never send one.
+      class FrontmatterChanged < OpenLanes::Error; end
+
+      # One file as read.
+      #
+      # @!attribute body [String] the raw bytes
+      # @!attribute etag [String] a strong, quoted ETag of those bytes
+      Document = Data.define(:body, :etag)
+
+      class << self
+        # @param body [String]
+        # @return [String] `"<sha256 hex>"`, quotes included, as HTTP sends it
+        def etag(body) = %("#{Digest::SHA256.hexdigest(body.b)}")
+      end
+
+      # @param plans_dir [String] the `.plans` directory
+      def initialize(plans_dir:)
+        @plans_dir = plans_dir
+      end
+
+      # @param ordinal [String] a plan number, as typed in the URL
+      # @return [Array<String>] readable names that exist, in {READABLE} order
+      # @raise [NotFound] when the ordinal is not a plan
+      def list(ordinal)
+        _, folder = folder_for(ordinal)
+        READABLE.select { File.file?(File.join(folder, it)) }
+      end
+
+      # @param ordinal [String]
+      # @param name [String]
+      # @return [Document]
+      # @raise [NotFound]
+      def read(ordinal, name)
+        body = File.binread(path_for(ordinal, name)).force_encoding(Encoding::UTF_8)
+        Document.new(body:, etag: self.class.etag(body))
+      end
+
+      # Replace a whole file, guarded three ways: an editable name, the
+      # ETag the page read, and the frontmatter block unchanged.
+      #
+      # @param ordinal [String]
+      # @param name [String]
+      # @param body [String] the whole new file
+      # @param if_match [String] the ETag the page read
+      # @return [String] the new ETag
+      # @raise [NotFound, ReadOnly, Stale, FrontmatterChanged]
+      def write(ordinal, name, body:, if_match:)
+        path = path_for(ordinal, name)
+        raise ReadOnly, "#{name} is read only" unless EDITABLE.include?(name)
+
+        current = read(ordinal, name)
+        raise Stale, "#{name} changed since it was read; reload it" unless current.etag == if_match.to_s.strip
+        raise FrontmatterChanged, "the frontmatter of #{name} is locked" unless frontmatter(current.body) == frontmatter(body)
+
+        replace(path, body)
+        self.class.etag(body)
+      end
+
+      private
+
+      # The raw frontmatter block, or nil when the file has none. Compared as
+      # bytes, not as parsed YAML, so a reordered key or a changed comment is
+      # a change too.
+      #
+      # @param content [String]
+      # @return [String, nil]
+      def frontmatter(content) = Frontmatter::PATTERN.match(content.b)&.then { it[1] }
+
+      # @param path [String]
+      # @param body [String]
+      # @return [void]
+      def replace(path, body)
+        temp = File.join(File.dirname(path), ".#{File.basename(path)}.#{SecureRandom.hex(6)}.tmp")
+        File.binwrite(temp, body)
+        File.rename(temp, path)
+      ensure
+        FileUtils.rm_f(temp) if temp && File.exist?(temp)
+      end
+
+      # @param ordinal [String]
+      # @param name [String]
+      # @return [String]
+      # @raise [NotFound]
+      def path_for(ordinal, name)
+        known = READABLE.find { it == name } or raise NotFound, "not a plan file: #{name}"
+        number, folder = folder_for(ordinal)
+        path = File.join(folder, known)
+        raise NotFound, "#{number} has no #{known}" unless File.file?(path)
+
+        path
+      end
+
+      # @param ordinal [String]
+      # @return [Array(Plans::Ordinal, String)] the plan number and its folder
+      # @raise [NotFound]
+      def folder_for(ordinal)
+        number = Plans::Ordinal.parse(ordinal.to_s) or raise NotFound, "not a plan number"
+        subject = Plans.tree(@plans_dir).find(number) or raise NotFound, "there is no plan #{number}"
+        [number, subject.feature.path]
+      end
+    end
+  end
+end

data/lib/openlanes/web/request_log.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require "rack/body_proxy"
+require "rack/utils"
+
+module OpenLanes
+  module Web
+    # One log entry per request, through {Log}, wrapped around the whole app
+    # so a request Rack::Protection or host authorization turns away is
+    # logged too.
+    #
+    # Not Sinatra's `logging` or Rack::CommonLogger: both write Apache-style
+    # text to `rack.errors`, which is the launcher's raw output file rather
+    # than the log the engine writes, and neither knows a page from a request.
+    #
+    # The event stream is the exception to "one entry per request". It stays
+    # open as long as the page does, so it is logged twice: when it opens, and
+    # when Puma closes its body, with how long it was open. Logged once, at
+    # the end, a page left open all day would not appear until evening.
+    #
+    # Static assets and `/healthz` are logged at debug: the launcher polls
+    # `/healthz` ten times a second while it waits, and a page load fetches a
+    # dozen assets, so at info they would bury the API calls.
+    class RequestLog
+      # A response of this type is a stream that outlives its request.
+      STREAM = "text/event-stream"
+
+      # Paths logged at info. Everything else is the page's own files.
+      API = %r{\A/api(/|\z)}
+
+      # @param app [#call]
+      # @param logger [#info, #debug, #error, nil] {Log.logger} when nil, read
+      #   per request so a {Log.setup} after this was built still applies
+      # @param clock [#call] monotonic seconds
+      def initialize(app, logger: nil, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
+        @app = app
+        @logger = logger
+        @clock = clock
+      end
+
+      # @param env [Hash] the Rack environment
+      # @return [Array] status, headers and body, the body wrapped for a stream
+      def call(env)
+        started = @clock.call
+        status, headers, body = @app.call(env)
+        return [status, headers, opened(env, status, started, body)] if stream?(headers)
+
+        write(level(env), env, "#{line(env)} #{status}", status:, started:)
+        [status, headers, body]
+      rescue StandardError => e
+        write(:error, env, "#{line(env)} raised #{e.class}", status: 500, started:, exception: e)
+        raise
+      end
+
+      private
+
+      # Logs the stream opening, and hands back a body that logs its close.
+      #
+      # @return [Rack::BodyProxy]
+      def opened(env, status, started, body)
+        write(:info, env, "#{line(env)} stream opened", status:)
+        Rack::BodyProxy.new(body) { write(:info, env, "#{line(env)} stream closed", status:, started:) }
+      end
+
+      # @param headers [Hash]
+      # @return [Boolean]
+      def stream?(headers)
+        type = headers["content-type"] || headers["Content-Type"]
+        type.to_s.start_with?(STREAM)
+      end
+
+      # @param env [Hash]
+      # @return [Symbol]
+      def level(env) = env["PATH_INFO"].to_s.match?(API) ? :info : :debug
+
+      # @param env [Hash]
+      # @return [String] e.g. "GET /api/board"
+      def line(env) = "#{env["REQUEST_METHOD"]} #{env["PATH_INFO"]}"
+
+      # The page's own id, from `?page=`, so the requests one browser tab
+      # made can be picked out of several tabs' worth.
+      #
+      # @param env [Hash]
+      # @return [String, nil]
+      def page(env)
+        id = Rack::Utils.parse_query(env["QUERY_STRING"].to_s)["page"]
+        id.is_a?(String) && !id.empty? ? id : nil
+      end
+
+      # @param level [Symbol]
+      # @param env [Hash]
+      # @param message [String]
+      # @param status [Integer]
+      # @param started [Float, nil] when given, the entry carries the duration since
+      # @param exception [Exception, nil]
+      # @return [void]
+      def write(level, env, message, status:, started: nil, exception: nil)
+        payload = { method: env["REQUEST_METHOD"], path: env["PATH_INFO"], status: status.to_i, page: page(env) }.compact
+        entry = { message:, payload:, exception: }.compact
+        entry[:duration] = ((@clock.call - started) * 1000).round(1) if started
+        (@logger || Log.logger).public_send(level, **entry)
+      rescue StandardError
+        # A log that cannot be written is no reason to fail the request.
+        nil
+      end
+    end
+  end
+end

data/lib/openlanes/web/server.rb

@@ -7,7 +7,8 @@ require "puma/server"
 module OpenLanes
   module Web
     # The app on Puma, multi-threaded because every open page holds one
-    # thread for its event stream, and bound to {HOST} only.
+    # thread for its event stream, and bound to {HOST} unless a {Listen}
+    # names other addresses.
     #
     #   server = Web.server(app:, context:, port: 11000).start
     #   server.url   # => "http://127.0.0.1:11000"
@@ -18,29 +19,31 @@ module OpenLanes
       MAX_THREADS = 64
 
       # Seconds Puma waits for open streams after {#stop}. They notice the
-      # context's `stopping` within a tick; this is the backstop.
+      # context's `stopping` on the `stop` nudge {#stop} publishes; this is
+      # the backstop, for a Redis that went away and took the nudge with it.
       SHUTDOWN_AFTER = 2
 
       # @param app [#call]
       # @param context [OpenLanes::Web::Context]
       # @param port [Integer] 0 for any free port
-      # @param host [String]
+      # @param listen [OpenLanes::Web::Listen] the addresses to listen on
       # @param log [IO] where Puma writes its errors
-      def initialize(app:, context:, port: DEFAULT_PORT, host: HOST, log: $stderr)
+      def initialize(app:, context:, port: DEFAULT_PORT, listen: Listen.parse(nil), log: $stderr)
         @app = app
         @context = context
         @port = Integer(port)
-        @host = host
+        @listen = listen
         @log = log
       end
 
-      # @return [String]
-      attr_reader :host
+      # @return [OpenLanes::Web::Listen]
+      attr_reader :listen
 
       # Listen, and serve on Puma's own threads.
       #
       # @return [self]
       # @raise [Errno::EADDRINUSE] when the port is taken
+      # @raise [Errno::EADDRNOTAVAIL] when an address is not this machine's
       def start
         @puma = Puma::Server.new(@app,
           nil,
@@ -48,17 +51,29 @@ module OpenLanes
           max_threads:          MAX_THREADS,
           force_shutdown_after: SHUTDOWN_AFTER,
           log_writer:           Puma::LogWriter.new(File.open(File::NULL, "w"), @log))
-        @puma.add_tcp_listener(@host, @port)
+        # Port 0 is resolved by the first listener, and the rest share it, so
+        # every address answers on the one port {#url} reports.
+        @listen.ips.each { @puma.add_tcp_listener(it, @puma.connected_ports.first || @port) }
         @thread = @puma.run
         @context.chat&.start
         self
+      rescue SystemCallError
+        # A later address failing leaves the earlier ones bound; Puma is not
+        # running yet, so close its sockets here rather than leak them to
+        # whoever retries in this process.
+        @puma&.binder&.close
+        @puma = nil
+        raise
       end
 
       # @return [Integer] the port listened on, the real one when 0 was asked
       def port = @puma&.connected_ports&.first || @port
 
-      # @return [String]
-      def url = "http://#{@host}:#{port}"
+      # @return [String] where this machine's browser opens the page
+      def url = @listen.browse_url(port)
+
+      # @return [Array<String>] every address the page answers on
+      def urls = @listen.urls(port)
 
       # Block until the server stops.
       #
@@ -70,6 +85,10 @@ module OpenLanes
       # @return [void]
       def stop
         @context.stopping.make_true
+        # Wakes every stream blocked waiting for news, here and on any other
+        # server following this project, so ours see `stopping` now rather
+        # than at their next ping. The others look, find nothing, and sleep.
+        Live::News.publish(store: @context.store, keys: @context.keys, kind: "stop")
         @context.chat&.stop
         @puma&.stop(true)
         @context.stores.close

data/lib/openlanes/web.rb

@@ -25,8 +25,8 @@ module OpenLanes
     # Where the server listens unless told otherwise (contract.md).
     DEFAULT_PORT = 11_000
 
-    # Loopback only: the page can stop agents, and nothing off this machine
-    # may reach it (W3).
+    # Loopback by default: the page can stop agents and has no login, so
+    # nothing off this machine reaches it unless `--ips` or `-I` says so (W3).
     HOST = "127.0.0.1"
 
     module_function
@@ -54,9 +54,11 @@ module OpenLanes
       Live::Chat.new(stores:, keys:, machine:, label: node&.label || machine, **)
     end
 
+    # The routes, with every request logged on the way out (see {RequestLog}).
+    #
     # @param context [OpenLanes::Web::Context]
-    # @return [Class] a Rack application
-    def app(context) = App.serving(context)
+    # @return [#call] a Rack application
+    def app(context) = RequestLog.new(App.serving(context))
 
     # @param app [#call]
     # @param context [OpenLanes::Web::Context]