openlanes 3.0.1 → 3.1.0

data/lib/openlanes/apps/definition.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Apps
+    # The `app:` block of `.openlanes.yml`: what kind of app a project builds
+    # and the processes that run it, in the order they start.
+    #
+    # Written by luke on a plan's first build (M12) and only ever read here.
+    # It is new surface that every project will copy, so anything it does not
+    # know is refused with the file named, exactly as {Config} refuses an
+    # unknown top-level key: a `restart-on:` typo silently ignored is a
+    # watcher that never fires and nobody can say why.
+    #
+    # @!attribute [r] kind
+    #   @return [String, nil] free text, `web` or `desktop` (M11); recorded,
+    #     not interpreted here
+    # @!attribute [r] processes
+    #   @return [Array<OpenLanes::Apps::ProcessSpec>] in start order
+    Definition = Data.define(:kind, :processes) do
+      # @return [OpenLanes::Apps::ProcessSpec, nil] the process marked
+      #   `preview: true`; nil when none is, which means no Preview tab
+      def preview = processes.find(&:preview?)
+
+      # @param name [String]
+      # @return [OpenLanes::Apps::ProcessSpec, nil]
+      def process(name) = processes.find { it.name == name.to_s }
+
+      # Every process the Preview tab's Restart backend button restarts:
+      # all but the one it shows, in declared order.
+      #
+      # @return [Array<OpenLanes::Apps::ProcessSpec>]
+      def backend = processes.reject(&:preview?)
+
+      # @return [Hash] as `.openlanes.yml` spells it
+      def to_h = { "kind" => kind, "processes" => processes.map(&:to_h) }.compact
+    end
+
+    class Definition
+      # Keys the `app:` mapping may hold.
+      KEYS = %w[kind processes].freeze
+
+      # Keys each process may hold.
+      PROCESS_KEYS = %w[name start ready restart_on preview].freeze
+
+      # A process name: what `PORT_<NAME>`, the Preview tab and `target:`
+      # all carry, so nothing a shell or a URL would trip over.
+      NAME = /\A[A-Za-z0-9][A-Za-z0-9_-]*\z/
+
+      class << self
+        # @param value [Object] the parsed `app:` value
+        # @param path [String] the file, for every error
+        # @return [OpenLanes::Apps::Definition]
+        # @raise [OpenLanes::Error] naming the file and what is wrong
+        def parse(value, path:)
+          fail_with(path, "app: must be a mapping with processes:") unless value.is_a?(Hash)
+
+          data = value.transform_keys(&:to_s)
+          unknown!(path, "app", data.keys - KEYS, KEYS)
+          kind = data["kind"]
+          fail_with(path, "app.kind: #{kind.inspect} is not a word like web") unless kind.nil? || kind.is_a?(String)
+
+          list = data["processes"]
+          fail_with(path, "app.processes: must be a list of at least one process") unless list.is_a?(Array) && !list.empty?
+
+          processes = list.each_with_index.map { |entry, index| process(entry, index, path) }
+          checked(new(kind: kind&.strip, processes:), path)
+        end
+
+        private
+
+        # @return [OpenLanes::Apps::ProcessSpec]
+        def process(entry, index, path)
+          where = "app.processes[#{index}]"
+          fail_with(path, "#{where}: must be a mapping with name: and start:") unless entry.is_a?(Hash)
+
+          data = entry.transform_keys(&:to_s)
+          unknown!(path, where, data.keys - PROCESS_KEYS, PROCESS_KEYS)
+          name = data["name"]
+          fail_with(path, "#{where}: needs name:, letters, digits, - or _") unless name.is_a?(String) && name.match?(NAME)
+
+          ProcessSpec.new(name:,
+            start:      start(data["start"], "app.processes.#{name}", path),
+            ready:      ready(data["ready"], "app.processes.#{name}", path),
+            restart_on: restart_on(data["restart_on"], "app.processes.#{name}", path),
+            preview:    preview(data["preview"], "app.processes.#{name}", path))
+        end
+
+        # @return [String]
+        def start(value, where, path)
+          return value.strip if value.is_a?(String) && !value.strip.empty?
+
+          fail_with(path, "#{where}: needs start:, the shell line that runs it")
+        end
+
+        # @return [String, nil]
+        def ready(value, where, path)
+          return nil if value.nil?
+          return value if value.is_a?(String) && value.start_with?("/")
+
+          fail_with(path, "#{where}.ready: #{value.inspect} is not an HTTP path like /up")
+        end
+
+        # @return [Array<String>]
+        def restart_on(value, where, path)
+          globs = value.is_a?(String) ? [value] : value
+          return [] if globs.nil?
+          return globs if globs.is_a?(Array) && globs.all? { it.is_a?(String) && !it.strip.empty? }
+
+          fail_with(path, "#{where}.restart_on: must be a glob or a list of globs")
+        end
+
+        # @return [Boolean]
+        def preview(value, where, path)
+          return value == true if [true, false, nil].include?(value)
+
+          fail_with(path, "#{where}.preview: #{value.inspect} is not true or false")
+        end
+
+        # Two processes that one name, one `PORT_` variable or two preview
+        # flags would confuse.
+        #
+        # @return [OpenLanes::Apps::Definition]
+        def checked(definition, path)
+          names = definition.processes.map(&:name)
+          twice = names.select { names.count(it) > 1 }.uniq
+          fail_with(path, "app.processes: #{twice.join(", ")} named twice") unless twice.empty?
+
+          envs = definition.processes.group_by(&:port_env).select { _2.size > 1 }
+          envs.each do |env, specs|
+            fail_with(path, "app.processes: #{specs.map(&:name).join(" and ")} would both be #{env}")
+          end
+
+          previews = definition.processes.select(&:preview?)
+          if previews.size > 1
+            fail_with(path, "app.processes: only one process may be preview: true, not #{previews.map(&:name).join(" and ")}")
+          end
+          definition
+        end
+
+        # @return [void]
+        def unknown!(path, where, unknown, known)
+          return if unknown.empty?
+
+          fail_with(path, "#{where}: unknown key#{"s" unless unknown.size == 1} #{unknown.join(", ")}; known: #{known.join(", ")}")
+        end
+
+        # @return [void] never returns
+        def fail_with(path, message) = raise(Error, "#{path}: #{message}")
+      end
+    end
+  end
+end

data/lib/openlanes/apps/framing.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+
+module OpenLanes
+  module Apps
+    # Whether the Preview tab may show an app in an iframe, or must open it in
+    # a window of its own (M13, plan 012 item 11).
+    #
+    # The page cannot find out: a cross-origin frame's DOM is closed to it
+    # and a refused frame still fires `load`. So the runner asks the app
+    # itself, once its preview process is ready and after each restart, and
+    # reads the two headers that refuse framing:
+    #
+    # - `X-Frame-Options`: `DENY` refuses everyone, and `SAMEORIGIN` refuses
+    #   the control room too, whose origin differs from the app's by port.
+    #   `ALLOW-FROM` is obsolete and ignored by browsers, so it frames.
+    # - `Content-Security-Policy: frame-ancestors`: refuses unless one of its
+    #   sources matches the control room's origin. `'self'` is the app's own
+    #   origin, so it refuses too.
+    #
+    # Either one refusing sends the tab to a window. Browsers let
+    # `frame-ancestors` override `X-Frame-Options`; this stays on the
+    # cautious side, because a window always works and a refused frame is a
+    # blank tab nobody can explain.
+    module Framing
+      # The control room's origin unless told otherwise.
+      ORIGIN = "http://#{Status::HOST}:11000".freeze
+
+      # Default ports, for a source that names none.
+      DEFAULT_PORTS = { "http" => 80, "https" => 443 }.freeze
+
+      module_function
+
+      # @param headers [Hash{String => String, Array<String>}] any case
+      # @param origin [String] the page that embeds it
+      # @return [Symbol] `:frame` or `:window`
+      def verdict(headers, origin: ORIGIN)
+        values = headers.to_h { |name, value| [name.to_s.downcase, Array(value).join(",")] }
+        refused = x_frame_refuses?(values["x-frame-options"]) || ancestors_refuse?(values["content-security-policy"], URI(origin))
+        refused ? :window : :frame
+      end
+
+      # One GET of the URL the tab would open, then {.verdict}. An app that
+      # cannot be asked opens in a window: it may work there, and an iframe
+      # would show nothing either way.
+      #
+      # @param url [String]
+      # @param origin [String]
+      # @return [Symbol]
+      def check(url, origin: ORIGIN)
+        uri = URI(url)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.open_timeout = Probe::TIMEOUT
+        http.read_timeout = Probe::TIMEOUT
+        response = http.request_get(uri.request_uri)
+        verdict(response.to_hash, origin:)
+      rescue SystemCallError, IOError, Timeout::Error, Net::HTTPBadResponse, SocketError, URI::Error
+        :window
+      end
+
+      # @param value [String, nil]
+      # @return [Boolean]
+      def x_frame_refuses?(value)
+        value.to_s.split(",").map { it.strip.upcase }.intersect?(%w[DENY SAMEORIGIN])
+      end
+
+      # Every policy is enforced, so any one whose `frame-ancestors` leaves
+      # the origin out refuses it. A policy without the directive allows
+      # framing: it has no `default-src` fallback.
+      #
+      # @param value [String, nil] one or more policies, comma-joined
+      # @param origin [URI::Generic]
+      # @return [Boolean]
+      def ancestors_refuse?(value, origin)
+        value.to_s.split(",").any? do |policy|
+          directive = policy.split(";").map(&:strip).find { it.downcase.start_with?("frame-ancestors") }
+          directive && directive.split[1..].none? { allows?(it, origin) }
+        end
+      end
+
+      # @param source [String] one source expression
+      # @param origin [URI::Generic]
+      # @return [Boolean]
+      def allows?(source, origin)
+        text = source.strip.downcase
+        return true if text == "*"
+        return text.delete_suffix(":") == origin.scheme if text.match?(/\A[a-z][a-z0-9+.-]*:\z/)
+        return false if text.start_with?("'")
+
+        host_allows?(text, origin)
+      end
+
+      # @return [Boolean]
+      def host_allows?(text, origin)
+        match = text.match(%r{\A(?:(?<scheme>[a-z][a-z0-9+.-]*)://)?(?<host>[^:/]+)(?::(?<port>\d+|\*))?(?:/.*)?\z})
+        return false unless match
+        return false if match[:scheme] && match[:scheme] != origin.scheme
+        return false unless host_matches?(match[:host], origin.host)
+
+        port = match[:port]
+        port == "*" || (port || DEFAULT_PORTS[match[:scheme] || origin.scheme]).to_i == origin.port
+      end
+
+      # @return [Boolean]
+      def host_matches?(pattern, host)
+        return host == pattern unless pattern.start_with?("*.")
+
+        host.end_with?(pattern.delete_prefix("*"))
+      end
+    end
+  end
+end

data/lib/openlanes/apps/listing.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "json"
+
+module OpenLanes
+  module Apps
+    # The apps running on this machine, as the control room reads them from
+    # Redis: one hash per machine (`Live::Keys#apps`), a field per plan, its
+    # value the plan's process statuses and a heartbeat.
+    #
+    # Per machine rather than per checkout, because the control room runs in
+    # the main checkout and an app runs in the plan's worktree; per machine
+    # rather than per repository, because `127.0.0.1:11101` means nothing on
+    # another one.
+    #
+    # A runner killed with KILL never removes its field, so a field whose
+    # heartbeat is older than {TTL} is dropped by whoever reads it, the way
+    # `Live.board` drops a node that stopped heartbeating.
+    module Listing
+      # Seconds a plan's entry outlives its last heartbeat.
+      TTL = 30
+
+      module_function
+
+      # @param store [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore]
+      # @param keys [OpenLanes::Live::Keys]
+      # @param plan [String]
+      # @param statuses [Array<OpenLanes::Apps::Status>]
+      # @param now [Time]
+      # @return [void]
+      def write(store:, keys:, plan:, statuses:, now: Time.now)
+        store.hset(keys.apps, plan, JSON.generate("heartbeat" => now.to_f, "processes" => statuses.map(&:to_h)))
+      end
+
+      # @return [void]
+      def remove(store:, keys:, plan:) = store.hdel(keys.apps, plan)
+
+      # Every live process, oldest plan first, in each plan's start order.
+      #
+      # @param store [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore]
+      # @param keys [OpenLanes::Live::Keys]
+      # @param now [Time]
+      # @return [Array<Hash>] `<app>` hashes, as {Status#to_h} wrote them
+      def read(store:, keys:, now: Time.now)
+        entries = store.hgetall(keys.apps).transform_values { parse(it) }
+        stale = entries.select { |_, entry| entry.nil? || now.to_f - entry["heartbeat"].to_f > TTL }.keys
+        store.hdel(keys.apps, *stale) unless stale.empty?
+        entries.except(*stale).sort.flat_map { |_, entry| Array(entry["processes"]) }
+      end
+
+      # @return [Hash, nil] nil for a field that is not ours to read
+      def parse(json)
+        data = JSON.parse(json.to_s)
+        data.is_a?(Hash) ? data : nil
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/openlanes/apps/ports.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "socket"
+require "time"
+
+module OpenLanes
+  module Apps
+    # Every port an app holds on this machine, in one file (plan 012).
+    #
+    # One file per machine, not per checkout: a worktree isolates files, not
+    # ports, and two worktrees of one app asking for 11100 each would both
+    # be told yes by anything that only looked at its own checkout.
+    #
+    # A port is reserved under an exclusive `flock` and also probed, both by
+    # connecting to it and by binding it: something that never registered
+    # here (a dev server started by hand, the control room) still holds it.
+    # The probe and the spawn that follows it leave a gap no lock covers, so
+    # {Supervisor} retries a start that lost that race.
+    #
+    # An entry belongs to the pid that reserved it. Once that process is
+    # gone the entry counts as free, which is how a runner killed with KILL
+    # gives its ports back without anyone cleaning up after it.
+    class Ports
+      # The first port an app is given (M12).
+      FLOOR = 11_100
+
+      # The last. Not configurable: per-plan ranges are a non-goal.
+      CEILING = 11_999
+
+      # Overrides where the registry lives; the suite and the Cypress run
+      # point it at a temp file.
+      FILE_ENV = "OPENLANES_PORTS_FILE"
+
+      # Every port in the range is held.
+      class Exhausted < Error; end
+
+      class << self
+        # `$OPENLANES_PORTS_FILE`, else `${XDG_STATE_HOME:-~/.local/state}/openlanes/ports.json`.
+        #
+        # @param env [Hash]
+        # @return [String]
+        def default_path(env = ENV)
+          return env[FILE_ENV] unless env[FILE_ENV].to_s.strip.empty?
+
+          state = env["XDG_STATE_HOME"].to_s.strip
+          File.join(state.empty? ? File.expand_path("~/.local/state") : state, "openlanes", "ports.json")
+        end
+
+        # Nothing answers on the port and it can be bound on 127.0.0.1. The
+        # connect catches a listener on another address, which macOS lets a
+        # SO_REUSEADDR bind on 127.0.0.1 sit beside.
+        #
+        # @param port [Integer]
+        # @return [Boolean]
+        def free?(port)
+          return false if answering?(port)
+
+          TCPServer.new(Status::HOST, port).close
+          true
+        rescue SystemCallError
+          false
+        end
+
+        # @param port [Integer]
+        # @return [Boolean]
+        def answering?(port) = Probe.listening?(port, timeout: 0.2)
+
+        # @param pid [Integer]
+        # @return [Boolean]
+        def alive?(pid)
+          Process.kill(0, pid.to_i)
+          true
+        rescue Errno::ESRCH, RangeError
+          false
+        rescue Errno::EPERM
+          true
+        end
+      end
+
+      # @param path [String] the registry file
+      # @param owner [Integer] whose reservations these are
+      # @param floor [Integer]
+      # @param ceiling [Integer]
+      # @param free [#call] a port to whether it can be taken
+      # @param alive [#call] a pid to whether it still runs
+      def initialize(path: self.class.default_path, owner: Process.pid, floor: FLOOR, ceiling: CEILING,
+        free: self.class.method(:free?), alive: self.class.method(:alive?))
+        @path = path
+        @owner = owner
+        @floor = floor
+        @ceiling = ceiling
+        @free = free
+        @alive = alive
+      end
+
+      # @return [String]
+      attr_reader :path
+
+      # The lowest port at or above the floor that nobody holds.
+      #
+      # @param root [String] the checkout the app runs in
+      # @param plan [String]
+      # @param process [String]
+      # @return [Integer]
+      # @raise [OpenLanes::Apps::Ports::Exhausted]
+      def reserve(root:, plan:, process:)
+        locked do |entries|
+          port = (@floor..@ceiling).find { !entries.key?(it) && @free.call(it) }
+          raise Exhausted, "every port from #{@floor} to #{@ceiling} is taken" unless port
+
+          entries[port] = { "owner" => @owner, "root" => root, "plan" => plan, "process" => process, "at" => Time.now.utc.iso8601 }
+          port
+        end
+      end
+
+      # @param port [Integer]
+      # @return [void]
+      def release(port) = locked { it.delete(port.to_i) }
+
+      # Everything one process holds.
+      #
+      # @param pid [Integer]
+      # @return [Array<Integer>] the ports released
+      def release_owner(pid = @owner)
+        locked do |entries|
+          entries.select { |_, entry| entry["owner"] == pid }.keys.tap { |ports| ports.each { entries.delete(it) } }
+        end
+      end
+
+      # Every live reservation, by port.
+      #
+      # @return [Hash{Integer => Hash}] `owner`, `root`, `plan`, `process`, `at`
+      def entries = locked(&:dup)
+
+      private
+
+      # Read, hand the live entries to the block, write back what it left.
+      # Dead owners' entries are dropped on the way.
+      #
+      # @yieldparam entries [Hash{Integer => Hash}]
+      # @return [Object] the block's value
+      def locked
+        FileUtils.mkdir_p(File.dirname(path))
+        File.open(path, File::RDWR | File::CREAT, 0o644) do |file|
+          file.flock(File::LOCK_EX)
+          entries = parse(file.read).select { |_, entry| @alive.call(entry["owner"]) }
+          yield(entries).tap do
+            file.rewind
+            file.truncate(0)
+            file.write(JSON.pretty_generate(entries.sort.to_h { [it.first.to_s, it.last] }))
+          end
+        end
+      end
+
+      # @return [Hash{Integer => Hash}]
+      def parse(text)
+        data = text.strip.empty? ? {} : JSON.parse(text)
+        data.is_a?(Hash) ? data.to_h { |port, entry| [port.to_i, entry] }.select { |_, entry| entry.is_a?(Hash) } : {}
+      rescue JSON::ParserError
+        # A file somebody broke by hand is a registry with nothing in it; the
+        # probes still keep a taken port from being handed out twice.
+        {}
+      end
+    end
+  end
+end

data/lib/openlanes/apps/probe.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "socket"
+
+module OpenLanes
+  module Apps
+    # Whether a process is ready (plan 012, decision 1): something accepts a
+    # TCP connection on its port, and, when its `ready:` names a path, a GET
+    # of that path answers 2xx or 3xx.
+    #
+    # Only ever on the port the process was given. An app that moved to
+    # another port (Vite without `--strictPort` does) therefore never
+    # becomes ready, which is how a moved port fails the start rather than
+    # leaving the Preview tab pointing at nothing.
+    module Probe
+      # Seconds a connect, and a read, may take.
+      TIMEOUT = 1
+
+      module_function
+
+      # @param port [Integer]
+      # @param path [String, nil]
+      # @return [Boolean]
+      def ready?(port, path: nil)
+        return false unless listening?(port)
+        return true if path.nil?
+
+        status(port, path).between?(200, 399)
+      end
+
+      # Something accepted a connection on the port. Asking the socket for
+      # its peer is the part that matters: behind a network filter (Little
+      # Snitch on macOS) `Socket.tcp` with `connect_timeout` returns a socket
+      # for a refused connect, and only `getpeername` then says EINVAL. Without
+      # it every port on the machine reads as taken.
+      #
+      # @param port [Integer]
+      # @param timeout [Numeric] seconds the connect may take
+      # @return [Boolean]
+      def listening?(port, timeout: TIMEOUT)
+        Socket.tcp(Status::HOST, port, connect_timeout: timeout) do |socket|
+          socket.remote_address
+          true
+        end
+      rescue SystemCallError, IOError, SocketError
+        false
+      end
+
+      # @param port [Integer]
+      # @param path [String]
+      # @return [Integer] the status, 0 when nothing answered
+      def status(port, path)
+        http = Net::HTTP.new(Status::HOST, port)
+        http.open_timeout = TIMEOUT
+        http.read_timeout = TIMEOUT
+        http.request_get(path).code.to_i
+      rescue SystemCallError, IOError, Timeout::Error, Net::HTTPBadResponse, SocketError
+        0
+      end
+    end
+  end
+end

data/lib/openlanes/apps/process_spec.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Apps
+    # One process of an app, as `.openlanes.yml` declares it under
+    # `app.processes` (M12).
+    #
+    # @!attribute [r] name
+    #   @return [String] `api`, `web`; unique within the app
+    # @!attribute [r] start
+    #   @return [String] a shell line, run by `sh -c` in the checkout, so
+    #     `$PORT_WEB` expands there
+    # @!attribute [r] ready
+    #   @return [String, nil] an HTTP path that must answer 2xx or 3xx; nil
+    #     means a TCP connect is enough
+    # @!attribute [r] restart_on
+    #   @return [Array<String>] `Dir.glob` patterns, relative to the checkout
+    # @!attribute [r] preview
+    #   @return [Boolean] the one process the Preview tab shows
+    ProcessSpec = Data.define(:name, :start, :ready, :restart_on, :preview) do
+      def initialize(name:, start:, ready: nil, restart_on: [], preview: false) = super
+
+      # `PORT_<NAME>`: upcased, anything not a letter or digit as `_`, so
+      # `api-v2` is `PORT_API_V2` and a shell can expand it.
+      #
+      # @return [String]
+      def port_env = "PORT_#{name.upcase.gsub(/[^A-Z0-9]/, "_")}"
+
+      # @return [Boolean]
+      def preview? = preview == true
+
+      # @return [Hash] as `.openlanes.yml` spells it, defaults left out
+      def to_h
+        { "name" => name, "start" => start, "ready" => ready,
+          "restart_on" => (restart_on unless restart_on.empty?), "preview" => (true if preview?) }.compact
+      end
+    end
+  end
+end