runlight 0.1.0 → 0.1.3

data/lib/runlight/importers/visits.rb

@@ -162,11 +162,13 @@ module Runlight
           store.db.run("DELETE FROM rl_events WHERE site = ? AND ts >= ? AND ts < ? AND kind IN ('pageview', 'event') AND session IN (#{imported})",
                        [site_id, from, to, site_id])
           # Visits of these days that kept no rows go too. Their rows would come within EVENT_TAIL_MS of the step,
-          # so the time bounds let the (site, ts) index find them, with no scan of every event.
+          # so the time bounds let the (site, ts) index find them, with no scan of every event. That reaches back
+          # EVENT_TAIL_MS too, for a visit an earlier import began before this step whose rows were all in it: a
+          # CSV imported again has new ids, so its new visit does not take that one's place.
           store.db.run(
             "DELETE FROM rl_sessions WHERE site = ? AND imported = 1 AND started_at >= ? AND started_at < ?
          AND id NOT IN (SELECT e.session FROM rl_events e WHERE e.site = ? AND e.ts >= ? AND e.ts < ?)",
-            [site_id, from, to, site_id, from, to + tail],
+            [site_id, from - tail, to, site_id, from - tail, to + tail],
           )
           hits.each do |h|
             counts["visits"] += 1 if write_event(store, site, h["ns"], h["hit"])
@@ -336,8 +338,15 @@ module Runlight
       # One batch of a CSV file, sorted oldest first by the dashboard. As with Umami, only rows from before
       # Runlight's own first visit, and within what the site keeps, are written. A batch can run again: its
       # time span is cleared first, so batches must not share a moment, which the dashboard sees to.
-      # Returns { "pageviews", "events", "visits", "skipped" }.
-      def import_csv_visits(runlight, site_id, rows)
+      #
+      # A CSV's visitor column can hold anything, an email or an IP address among them, and a plain hash of
+      # one is easily undone by trying every address. So each key is an HMAC under a random secret made for
+      # the import and never stored: the first batch gets one, the answer carries it, and the caller sends
+      # it with every later batch of the same file, so a visit that runs across two batches stays one. A file
+      # imported again gets a new secret and new ids; its time span is cleared first, as always, so nothing is
+      # counted twice.
+      # Returns { "pageviews", "events", "visits", "skipped", "secret" }.
+      def import_csv_visits(runlight, site_id, rows, secret = nil)
         runlight.init
         raise ImportError.new("Unknown site", "unknown_site") if runlight.site(site_id).nil?
         if !rows.is_a?(Array) || rows.length > CsvVisits::CSV_BATCH
@@ -356,6 +365,7 @@ module Runlight
         format = CsvVisits.csv_format(clean.empty? ? [] : clean[0].keys)
         raise ImportError.new("This CSV is not an Umami export or Runlight's visit format", "import_csv_format") if format.nil?
 
+        key = secret.is_a?(String) && secret.match?(/\A[A-Za-z0-9_-]{22,128}\z/) ? secret : Accounts::Crypto.base64url(Accounts::Crypto.random_bytes(24))
         cutoff = runlight.retention_cutoff(site_id) || 0
         own = runlight.store.first_own_visit(site_id)
         finish = [own.nil? ? Float::INFINITY : own, runlight.now].min
@@ -369,10 +379,11 @@ module Runlight
         end
         hits = hits.each_with_index.sort_by { |h, i| [h["hit"]["ts"], i] }.map(&:first)
         skipped = clean.length - hits.length
-        return { "pageviews" => 0, "events" => 0, "visits" => 0, "skipped" => skipped } if hits.empty?
+        return { "pageviews" => 0, "events" => 0, "visits" => 0, "skipped" => skipped, "secret" => key } if hits.empty?
 
+        hits.each { |h| h["hit"]["key"] = Accounts::Crypto.base64url(Accounts::Crypto.hmac("SHA-256", key, h["hit"]["key"])) }
         counts = write_step(runlight, site_id, hits[0]["hit"]["ts"], hits[-1]["hit"]["ts"] + 1, hits)
-        counts.merge("skipped" => skipped)
+        counts.merge("skipped" => skipped, "secret" => key)
       end
 
       private_class_method :progress_key, :all, :write_step, :referrer_of, :from_umami, :write_event

data/lib/runlight/importers/write.rb

@@ -6,7 +6,9 @@ module Runlight
     module Write
       # Domains run by the shorteners themselves. Links there stay on Runlight's own path.
       SHORTENER_DOMAINS = %w[bit.ly bitly.com j.mp dub.sh dub.co dub.link short.gy rebrand.ly rebrandly.com rb.gy].freeze
-      private_constant :SHORTENER_DOMAINS
+      # The most clicks a day one imported link gets from a daily count.
+      MAX_DAILY = 10_000
+      private_constant :SHORTENER_DOMAINS, :MAX_DAILY
 
       # Browser and system names as other tools write them, in Runlight's spelling.
       BROWSERS = {
@@ -76,8 +78,10 @@ module Runlight
       # importing again skips what is already there.
       #
       # foreign: { "sourceId", "slug", "domain", "name", "url", "createdAt" }; history: { "clicks"?, "daily"? }.
+      # link_domain: given a branded domain, the domain as the site keeps it once it passes a link domain's checks,
+      # or nil to bring the link over without it.
       # Returns { "status", "clicks", "reason"?, "code"?, "params"? }.
-      def write_link(runlight, site, source, foreign, history)
+      def write_link(runlight, site, source, foreign, history, link_domain = ->(_domain) {})
         source_id = Js.string(foreign["sourceId"])
         id = imported_link_id(source, source_id)
         return { "status" => "skipped", "clicks" => 0 } unless runlight.store.link_by_id(id).nil?
@@ -96,8 +100,18 @@ module Runlight
                    "params" => { "slug" => slug } }
         end
 
-        domain = Sources.strip_www(Js.truthy?(foreign["domain"]) ? foreign["domain"].to_s : "")
-        domain = "" if SHORTENER_DOMAINS.include?(domain)
+        # The destination passes the checks a link made by hand does.
+        begin
+          destination = Links.clean_url(foreign["url"])
+        rescue LinkError => e
+          return { "status" => "failed", "clicks" => 0, "reason" => e.message, "code" => e.code, "params" => e.params }
+        end
+
+        domain = Sources.strip_www(Js.lower(Js.truthy?(foreign["domain"]) ? foreign["domain"].to_s : ""))
+        # A branded domain must pass the checks one added in Settings does, so an import can never claim the
+        # dashboard's own host or a name inside a private network. One that fails is left off: the link is
+        # served from the app's own path instead.
+        domain = domain.empty? || SHORTENER_DOMAINS.include?(domain) ? "" : (link_domain.call(domain) || "")
         now = runlight.now
         clicks = 0
         # Nothing in the transaction is one link's own problem (those are checked above),
@@ -173,7 +187,8 @@ module Runlight
             start = Client.parse_date("#{d["day"]}T00:00:00Z")
             next if (start.is_a?(Float) && !start.finite?) || !(d["clicks"].is_a?(Numeric) && d["clicks"].positive?)
 
-            n = [d["clicks"], 1_000_000].min
+            # A day's count is capped, so one made-up figure from a source cannot write millions of rows.
+            n = [d["clicks"], MAX_DAILY].min.floor
             i = 0
             while i < n
               store.insert_event({
@@ -194,7 +209,7 @@ module Runlight
             "domain" => domain,
             "slug" => slug,
             "name" => Js.slice(name, 0, 100),
-            "url" => foreign["url"],
+            "url" => destination,
             "createdAt" => created,
             "updatedAt" => created,
           })

data/lib/runlight/links.rb

@@ -21,6 +21,19 @@ module Runlight
       SecureRandom.random_bytes(6).bytes.map { |byte| ALPHABET[byte % ALPHABET.length] }.join
     end
 
+    # A link's destination as it is kept: a full http or https URL, up to 2,000 characters.
+    def self.clean_url(value)
+      text = Js.trim(Js.string(value.nil? ? "" : value))
+      url = Http::Url.parse(text)
+      raise LinkError.new("The destination must be a full URL, starting with https://", "link_url") if url.nil?
+      if url.protocol != "https:" && url.protocol != "http:"
+        raise LinkError.new("The destination must start with http:// or https://", "link_protocol")
+      end
+      raise LinkError.new("The destination is longer than 2,000 characters", "link_long") if Js.length(text) > 2000
+
+      url.href
+    end
+
     def create(site, input)
       input = Options.normalize(input)
       @runlight.init
@@ -109,15 +122,7 @@ module Runlight
     private
 
     def clean_url(value)
-      text = Js.trim(Js.string(value.nil? ? "" : value))
-      url = Http::Url.parse(text)
-      raise LinkError.new("The destination must be a full URL, starting with https://", "link_url") if url.nil?
-      if url.protocol != "https:" && url.protocol != "http:"
-        raise LinkError.new("The destination must start with http:// or https://", "link_protocol")
-      end
-      raise LinkError.new("The destination is longer than 2,000 characters", "link_long") if Js.length(text) > 2000
-
-      url.href
+      self.class.clean_url(value)
     end
 
     def default_name(url)

data/lib/runlight/mail/ses.rb

@@ -79,13 +79,14 @@ module Runlight
         })
         headers.delete("host")
         begin
-          response = (fetcher || Http::NetFetcher.new).fetch(url.href, { "method" => "POST", "headers" => headers, "body" => body, "timeoutMs" => 20_000 })
+          response = (fetcher || Http::NetFetcher.new).fetch(url.href, { "method" => "POST", "headers" => headers, "body" => body, "timeoutMs" => 20_000,
+                                                                            "maxBytes" => Transports::MAX_ERROR_BYTES + 1, "truncate" => true })
         rescue Http::FetchError => e
           raise MailError.new("Could not reach Amazon SES: #{e.message}", "mail_unreachable", { "host" => "Amazon SES", "detail" => e.message })
         end
         return if response.ok?
 
-        message = Transports.service_message(response.text)
+        message = Transports.service_message(Transports.error_text(response))
         raise MailError.new("Amazon SES answered #{response.status}#{message.empty? ? "" : ": #{message}"}", "mail_refused",
                             { "host" => "Amazon SES", "detail" => "#{response.status}#{message.empty? ? "" : " #{message}"}" })
       end

data/lib/runlight/mail/smtp.rb

@@ -83,6 +83,8 @@ module Runlight
         s.connect(security == "tls", REPLY_TIMEOUT_MS)
         expect = lambda do |codes, what|
           reply = s.next_reply(REPLY_TIMEOUT_MS)
+          # Something that is not a mail server gets no more than that said, so its own words are never passed on.
+          raise MailError, "SMTP #{what}: #{s.host}:#{s.port} did not answer as a mail server does" unless reply["code"].between?(200, 599)
           raise MailError, Transports.slice16("SMTP #{what}: #{reply["code"]} #{reply["text"]}", 300) unless codes.include?(reply["code"])
 
           reply

data/lib/runlight/mail/smtp_session.rb

@@ -9,6 +9,8 @@ module Runlight
     # (multi-line included, one at a time), and the whole send's deadline, which
     # every wait is held to.
     class SmtpSession
+      attr_reader :host, :port
+
       # deadline: nanoseconds on Smtp.monotonic's clock.
       def initialize(host, port, deadline, late_message)
         @host = host
@@ -20,12 +22,22 @@ module Runlight
         @lines = []
       end
 
+      # A mail server may be on this computer or a private network, but never at a link-local or cloud metadata
+      # address. The name is resolved here and the connection goes to the addresses checked, so it cannot
+      # answer differently a moment later.
       def connect(tls, timeout_ms)
         wait = [timeout_ms, left].min
         raise late if wait <= 0
 
+        bare = @host.sub(/\A\[/, "").sub(/\]\z/, "")
+        addresses = Safefetch.metadata_address?(bare) ? [bare] : Safefetch.lookup(bare)
+        if addresses.any? { |address| Safefetch.metadata_address?(address) }
+          raise MailError.new("SMTP: #{@host} is a link-local or cloud metadata address, which Runlight never connects to", "address_refused", { "host" => @host })
+        end
+
         begin
-          socket = Socket.tcp(@host, @port, connect_timeout: wait / 1000.0)
+          # A name that does not resolve is still tried by name, for the resolver's own words.
+          socket = open_socket(addresses.empty? ? [@host] : addresses, wait)
         rescue SystemCallError, SocketError, IO::TimeoutError => e
           raise late if left <= 0
 
@@ -192,6 +204,15 @@ module Runlight
         @socket = ssl
       end
 
+      # A connection to the first of the addresses that answers, each tried in the time left.
+      def open_socket(targets, wait)
+        targets.each_with_index do |target, i|
+          return Socket.tcp(target, @port, connect_timeout: [wait, left].min / 1000.0)
+        rescue SystemCallError, SocketError, IO::TimeoutError
+          raise if i == targets.length - 1 || left <= 0
+        end
+      end
+
       def connect_detail(error, whole_wait)
         return(whole_wait ? "timed out" : "connection failed") if error.is_a?(IO::TimeoutError) || error.is_a?(Errno::ETIMEDOUT)
 

data/lib/runlight/mail/transports.rb

@@ -81,17 +81,34 @@ module Runlight
         ""
       end
 
+      # The most of a service's error read, so an answer without end never fills memory. One byte more is asked of
+      # the fetcher, which hands back what it read, so that an answer past the limit is seen as such and read as empty.
+      MAX_ERROR_BYTES = 65_536
+
+      # A service's error text, or none when it runs past MAX_ERROR_BYTES.
+      def error_text(response)
+        Body.read_text_capped(response, MAX_ERROR_BYTES)
+      rescue Http::BodyTooLong
+        ""
+      end
+
       # POSTs to a service, with the errors the dashboard shows. init: "headers" and "body".
       def post(fetcher, url, init, explains = true)
         begin
-          response = fetcher.fetch(url, { "method" => "POST", "headers" => init["headers"], "body" => init["body"], "timeoutMs" => 20_000 })
+          # A webhook can be at any address, so it and each redirect are checked, as for every service.
+          response = Safefetch.service_fetch(url, { "method" => "POST", "headers" => init["headers"], "body" => init["body"], "timeoutMs" => 20_000,
+                                                     "maxBytes" => MAX_ERROR_BYTES + 1, "truncate" => true }, fetcher)
+        rescue MetadataAddressError => e
+          raise MailError.new(e.message, "address_refused", { "host" => e.host })
+        rescue RedirectRefusedError => e
+          raise MailError.new(e.message, "redirect_refused", { "host" => e.host, "to" => e.to })
         rescue Http::FetchError => e
           host = Http::Url.new(url).host
           raise MailError.new("Could not reach #{host}: #{e.message}", "mail_unreachable", { "host" => host, "detail" => e.message })
         end
         return if response.ok?
 
-        message = explains ? service_message(response.text) : ""
+        message = explains ? service_message(error_text(response)) : ""
         host = Http::Url.new(url).host
         raise MailError.new("#{host} answered #{response.status}#{message.empty? ? "" : ": #{message}"}", "mail_refused",
                             { "host" => host, "detail" => "#{response.status}#{message.empty? ? "" : " #{message}"}" })

data/lib/runlight/mcp.rb

@@ -28,6 +28,8 @@ module Runlight
     ].freeze
     # query.ts's MAX_FILTERS.
     MAX_FILTERS = 6
+    # The most messages one batch may hold.
+    MAX_BATCH = 20
 
     RANGE_KEYS = %w[site period from to filters].freeze
     COMPARE_KEYS = [*RANGE_KEYS, "compare", "compare_from", "compare_to"].freeze
@@ -321,8 +323,13 @@ module Runlight
       parsed, body = Js.parse_json(request.text)
       return Http::Response.new(Json.encode(rpc_error(nil, -32_700, "Send a JSON-RPC message")), status: 400, headers: headers) if !parsed || !Js.object?(body)
 
-      # Batches were in the 2025-03-26 protocol; answering them costs nothing.
+      # Batches were in the 2025-03-26 protocol. Only a short list, so one request cannot start hundreds of
+      # reports; they run one after another here, where TypeScript runs four at a time.
       if body.is_a?(Array)
+        if body.length > MAX_BATCH
+          return Http::Response.new(Json.encode(rpc_error(nil, -32_600, "Send at most #{MAX_BATCH} messages in a batch")), status: 400, headers: headers)
+        end
+
         answers = body.map { |message| answer(message, read_api) }.compact
         return answers.empty? ? Http::Response.new("", status: 202) : Http::Response.new(Json.encode(answers), status: 200, headers: headers)
       end

data/lib/runlight/metadata_address_error.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Runlight
+  # Refused before anything was fetched, because the address is link-local or a cloud's metadata service.
+  class MetadataAddressError < RuntimeError
+    attr_reader :host
+
+    def initialize(host)
+      @host = host
+      super("#{host} is a link-local or cloud metadata address, which Runlight never connects to")
+    end
+  end
+end

data/lib/runlight/oauth.rb

@@ -16,7 +16,10 @@ module Runlight
   # The token is an ordinary API token, so it appears in Settings, API and AI,
   # beside the others, and deleting it there disconnects the app. It reads
   # stats, or with the "manage" scope (asked for by a Runlight hub) it also
-  # changes one site's settings.
+  # changes one site's settings. With the "cms" scope (asked for by the
+  # WordPress, Drupal, and Craft plugins) it is the site's dashboard key, which
+  # only opens its read-only dashboard in the CMS, and the answer also carries
+  # the site's agents key, so connecting fills in the plugin's settings.
   #
   # The context the routes pass is a Hash: "runlight", "base", "isOwner" (callable(request) giving a bool), and
   # optionally "signIn" (String), "isReader" (callable(request) giving a bool), "accountOf" (callable(request)
@@ -29,6 +32,7 @@ module Runlight
     REGISTRATIONS_PER_MINUTE = 10
     # The longest client id, which carries the app's name and redirect addresses.
     MAX_CLIENT_ID = 2048
+    SCOPES = %w[read manage cms].freeze
 
     # Where the per-address count of registrations is kept. TypeScript keeps it in memory per install; here it
     # lives in the install's settings, keyed by a hash of the address, so every process shares it.
@@ -41,7 +45,7 @@ module Runlight
     }.freeze
 
     ESCAPES = { "&" => "&amp;", "<" => "&lt;", ">" => "&gt;", '"' => "&quot;", "'" => "&#39;" }.freeze
-    private_constant :REGISTRATIONS, :CORS, :ESCAPES
+    private_constant :REGISTRATIONS, :CORS, :ESCAPES, :SCOPES
 
     module_function
 
@@ -50,6 +54,16 @@ module Runlight
       base64url(OpenSSL::Digest::SHA256.digest(verifier))
     end
 
+    # The key a CMS plugin reports AI agents reading a site's pages with: made on first ask, or a new one that replaces it.
+    def agents_key(runlight, site, replace = false)
+      name = "observe-key:#{site}"
+      return runlight.store.setting_or_add(name) { "rlo_#{Hashing.random_id(20)}" } unless replace
+
+      key = "rlo_#{Hashing.random_id(20)}"
+      runlight.store.set_setting(name, key)
+      key
+    end
+
     # The URL that a 401 from the MCP endpoint points clients at, to start OAuth.
     def resource_metadata_url(origin, base)
       "#{origin}#{base}/.well-known/oauth-protected-resource"
@@ -71,7 +85,7 @@ module Runlight
       end
 
       if known.start_with?("/.well-known/oauth-protected-resource")
-        return json({ "resource" => "#{issuer}/mcp", "authorization_servers" => [issuer], "scopes_supported" => %w[read manage], "bearer_methods_supported" => ["header"] })
+        return json({ "resource" => "#{issuer}/mcp", "authorization_servers" => [issuer], "scopes_supported" => SCOPES, "bearer_methods_supported" => ["header"] })
       end
       if known.start_with?("/.well-known/oauth-authorization-server") || known.start_with?("/.well-known/openid-configuration")
         return json({
@@ -83,7 +97,7 @@ module Runlight
           "grant_types_supported" => ["authorization_code"],
           "code_challenge_methods_supported" => ["S256"],
           "token_endpoint_auth_methods_supported" => ["none"],
-          "scopes_supported" => %w[read manage],
+          "scopes_supported" => SCOPES,
         })
       end
 
@@ -141,8 +155,15 @@ module Runlight
           return refuse.call({ "error" => "invalid_request", "error_description" => "PKCE with S256 is required" })
         end
 
-        manage = (form.get("scope") || "").split(/[#{Js::SPACE}]+/o).include?("manage")
+        scopes = (form.get("scope") || "").split(/[#{Js::SPACE}]+/o)
+        cms = scopes.include?("cms")
+        manage = !cms && scopes.include?("manage")
         name = client["name"].to_s
+        # A CMS plugin is for the site it runs on: one of the sites counting its admin's domain, or a new one for it.
+        host = site_host(redirect)
+        # As the tracker counts: a site that lists the domain, or the only site here when it lists none, which counts any.
+        local = runlight.sites.select { |s| runlight.remote(s["id"]).nil? }
+        counting = local.length == 1 && local[0]["hostnames"].empty? ? local : local.select { |s| s["hostnames"].include?(host) }
 
         unless ctx["isOwner"].call(request)
           # Someone signed in who may only read would be sent to sign in again and again.
@@ -169,6 +190,27 @@ module Runlight
           end
           # The app names itself, so the page also shows where the answer goes, which it cannot fake.
           sends_to = "<p class=\"note\">Allowing sends you back to <strong>#{esc(Http::Url.new(redirect).host)}</strong>. Only allow it if you started connecting there.</p>"
+          if cms
+            if counting.empty? && !runlight.managed_sites
+              return page("Add this site first",
+                          "<p>No site in this Runlight counts <strong>#{esc(host)}</strong>. Add it to a site's domains in your code, then connect again from #{esc(name)}.</p>", 400)
+            end
+
+            choices = +""
+            counting.each { |s| choices << "<option value=\"#{esc(s["id"])}\">#{esc(s["name"])}</option>" }
+            choices << "<option value=\"new\">A new site for #{esc(host)}</option>" if runlight.managed_sites && counting.empty?
+            return page(
+              "Connect #{esc(name)}",
+              "<p><strong>#{esc(name)}</strong> wants to count <strong>#{esc(host)}</strong> in this Runlight.</p>\n" \
+              "<p>It will get the site's agents key, to report AI agents such as ChatGPT and Claude reading its pages, and a dashboard key, to show the site's " \
+              "read-only dashboard in its admin pages. It cannot change anything here or read other sites.</p>\n" \
+              "#{sends_to}\n" \
+              "<form method=\"post\" action=\"#{esc(base)}/oauth/authorize\">#{hidden}\n" \
+              "<label>Site<select name=\"site\">#{choices}</select></label>\n" \
+              "<p class=\"note\">Its dashboard key appears in Settings, API and AI, where deleting it stops the dashboard in #{esc(name)}.</p>\n" \
+              '<div class="buttons"><button type="submit" name="decision" value="deny" class="ghost">Deny</button><button type="submit" name="decision" value="allow">Allow</button></div></form>',
+            )
+          end
           sites = runlight.sites
           if manage
             # Changing settings is for one site at a time, so there is no "every site" here.
@@ -209,6 +251,22 @@ module Runlight
         return back.call({ "error" => "access_denied" }) if form.get("decision") != "allow"
 
         site = form.get("site") || ""
+        if cms
+          if site == "new"
+            begin
+              site = if !counting.empty? then counting[0]["id"]
+                     elsif runlight.managed_sites then runlight.add_site({ "hostnames" => [host] })["id"]
+                     else ""
+                     end
+            rescue RangeError => e
+              # Such as an address, which a site cannot count by: there is no new site to make.
+              return back.call({ "error" => "invalid_request", "error_description" => e.message })
+            end
+          elsif counting.none? { |s| s["id"] == site }
+            site = ""
+          end
+          return back.call({ "error" => "invalid_request", "error_description" => "Pick a site that counts #{host}" }) if site == ""
+        end
         return back.call({ "error" => "invalid_request", "error_description" => "Unknown site" }) if site != "" && runlight.site(site).nil?
         if manage && (site == "" || !runlight.remote(site).nil?)
           return back.call({ "error" => "invalid_request", "error_description" => "Pick the site to manage" })
@@ -216,7 +274,7 @@ module Runlight
 
         code = Hashing.random_id(32)
         by = ctx["accountOf"]&.call(request)
-        grant = { "client" => client_id, "redirect" => redirect, "challenge" => challenge, "site" => site, "scope" => manage ? "manage" : "read",
+        grant = { "client" => client_id, "redirect" => redirect, "challenge" => challenge, "site" => site, "scope" => cms ? "cms" : manage ? "manage" : "read",
                   "expires" => runlight.now + CODE_MS }
         grant["by"] = by if !by.nil? && by != ""
         runlight.store.set_setting("oauth-code:#{Hashing.sha256(code)}", Json.encode(grant))
@@ -230,9 +288,8 @@ module Runlight
         return oauth_error("unsupported_grant_type", "Only authorization_code is supported") if form.get("grant_type") != "authorization_code"
 
         key = "oauth-code:#{Hashing.sha256(form.get("code") || "")}"
-        stored = runlight.store.setting(key)
-        # A code works once: it is gone before anything else is checked.
-        runlight.store.set_setting(key, nil) if !stored.nil? && stored != ""
+        # A code works once: it is gone before anything else is checked, and of two swaps at once only one gets it.
+        stored = runlight.store.take_setting(key)
         grant = !stored.nil? && stored != "" ? Json.decode(stored) : nil
         return oauth_error("invalid_grant", "The code has expired or was already used") if grant.nil? || grant["expires"] < runlight.now
         if grant["client"] != form.get("client_id") || grant["redirect"] != form.get("redirect_uri")
@@ -249,12 +306,16 @@ module Runlight
           runlight.store.set_setting(found["usedKey"], stored_client ? Json.encode(used) : runlight.now.to_s)
         end
         secret = "rl_#{Hashing.random_id(20)}"
-        scope = grant["scope"] == "manage" ? "manage" : "read"
+        scope = if grant["scope"] == "cms" then "cms"
+                elsif grant["scope"] == "manage" then "manage"
+                else "read"
+                end
+        # A CMS plugin's token is the site's dashboard key, which only opens its read-only dashboard there.
         row = {
           "id" => Hashing.random_id,
           "name" => Js.slice("#{Js.string(client["name"].nil? ? "An app" : client["name"])} (OAuth)", 0, 100),
           "site" => grant["site"].to_s,
-          "scope" => scope,
+          "scope" => scope == "cms" ? "embed" : scope,
           "hash" => Hashing.sha256(secret),
           "hint" => secret[-4..],
           "createdAt" => runlight.now,
@@ -268,6 +329,11 @@ module Runlight
         end
         # A hub's own address, from where it asked to be sent back, so the picker only ever sends choices there.
         runlight.store.set_setting("token-origin:#{row["id"]}", Http::Url.new(grant["redirect"].to_s).origin) if scope == "manage"
+        # A CMS plugin gets the site's agents key too, so connecting fills in every key its settings ask for.
+        if scope == "cms"
+          return json({ "access_token" => secret, "token_type" => "Bearer", "scope" => scope, "site" => grant["site"],
+                        "agents_key" => agents_key(runlight, grant["site"].to_s) })
+        end
         # site is not part of OAuth, but a hub needs to know which site it was given.
         answer = { "access_token" => secret, "token_type" => "Bearer", "scope" => scope }
         answer["site"] = grant["site"] if grant["site"] != ""
@@ -287,12 +353,8 @@ module Runlight
 
     # The key client ids are signed with, made on first use and kept in the database for every process.
     def client_key(runlight)
-      saved = runlight.store.setting("oauth-key")
-      return saved if !saved.nil? && saved != ""
-
-      made = Hashing.random_id(32)
-      runlight.store.set_setting("oauth-key", made)
-      made
+      # Written only if no other request wrote one first, so every process signs with the same key.
+      runlight.store.setting_or_add("oauth-key") { Hashing.random_id(32) }
     end
 
     # The app a client id names, and where to note that it connected: {"client" => ..., "usedKey" => ...}, or nil.
@@ -315,6 +377,11 @@ module Runlight
       { "client" => client, "usedKey" => used_key }
     end
 
+    # A redirect address's host as a site's domains are kept: lowercase, with no port and no www.
+    def site_host(redirect)
+      Http::Url.new(redirect).hostname.downcase.sub(/\.+\z/, "").sub(/\Awww\./, "")
+    end
+
     def constant_time_equal(a, b)
       a.bytesize == b.bytesize && OpenSSL.fixed_length_secure_compare(a, b)
     end
@@ -406,7 +473,7 @@ module Runlight
       )
     end
 
-    private_class_method :base64url, :from_base64url, :client_key, :client_for, :constant_time_equal, :esc, :json, :oauth_error,
+    private_class_method :base64url, :from_base64url, :client_key, :client_for, :site_host, :constant_time_equal, :esc, :json, :oauth_error,
                          :allowed_redirect, :json_form, :allow_registration, :register, :page
   end
 end

data/lib/runlight/rack_app.rb

@@ -5,9 +5,10 @@ module Runlight
   # in the order the standalone server answers: a link domain added in Settings
   # first (it leaves the dashboard's own paths alone), then `{linkPath}/{slug}`
   # on the app's own domain, then the dashboard and API under the routes' base
-  # path. As middleware, every other request goes on to the app, after AI agent
-  # fetches among them are recorded (observe), which returns at once for
-  # everyone else.
+  # path. As middleware, every other request goes on to the app, a short link
+  # with no link by that name included, so the app's own 404 page answers it,
+  # after AI agent fetches among them are recorded (observe), which returns at
+  # once for everyone else. Alone, Runlight answers those with its own 404.
   #
   #   # config.ru, Runlight alone
   #   run Runlight::RackApp.new(runlight: RL, base_path: "/runlight")
@@ -33,6 +34,8 @@ module Runlight
       @observe = observe
       @route_options = route_options
       @lock = Mutex.new
+      @warming = nil
+      @warm_failed_at = nil
     end
 
     def core
@@ -51,10 +54,16 @@ module Runlight
       rescue Http::Request::TooLarge
         # The rest is never read, so the connection is not fit for another request.
         return [413, { "content-type" => "application/json; charset=utf-8", "connection" => "close" }, [TOO_LARGE]]
+      rescue StandardError => e
+        raise if @app.nil?
+
+        # Whatever Runlight cannot read, the app behind still answers.
+        warn "Runlight: #{e.message}"
+        return @app.call(env)
       end
       response = answer(request)
       if response.nil?
-        core.observe(request) if @observe && request.method == "GET"
+        observe(request) if @observe && request.method == "GET"
         return @app.call(env)
       end
       status, headers, body = response.to_rack
@@ -63,21 +72,35 @@ module Runlight
     end
 
     # The answer to one request, or nil when it is not Runlight's and an app sits behind.
+    #
+    # The app's own requests never wait on Runlight: until Runlight has made its tables they go straight to
+    # the app while init runs beside them (a link domain is known only once it has), and anything that goes
+    # wrong in Runlight is logged and the request still reaches the app.
     def answer(request)
       context = { "ip" => request.remote_address }
-      linked = core.link_domain_response(request, context)
-      return linked unless linked.nil?
-
       path = Http::Url.parse(request.url)&.pathname || "/"
       link_path = core.link_path
-      if request.method == "GET" && path.match?(%r{\A#{Regexp.escape(link_path)}/[^/]+/?\z})
-        return core.link_handler.call(request, context)
+      link = request.method == "GET" && path.match?(%r{\A#{Regexp.escape(link_path)}/[^/]+/?\z})
+      # The discovery documents OAuth clients read at the site's root are answered too, as under the base path.
+      own = @app.nil? || mine?(path) || link || Routes.oauth_document?(path)
+      if own || ready?
+        linked = core.link_domain_response(request, context)
+        return linked unless linked.nil?
+      end
+      if link
+        linked = core.link_handler.call(request, context)
+        # A slug with no link is the app's to answer, so its own 404 page shows.
+        return linked unless @app && linked.status == 404
+
+        own = mine?(path) || Routes.oauth_document?(path)
       end
-      return routes.handle(request) if @app.nil? || mine?(path)
+      return routes.handle(request) if own
 
       nil
     rescue StandardError => e
       warn "Runlight: #{e.message}"
+      return nil if @app && !own
+
       Http::Response.new(INTERNAL, status: 500, headers: {
                            "content-type" => "application/json; charset=utf-8",
                            "cache-control" => "no-store",
@@ -87,6 +110,37 @@ module Runlight
 
     private
 
+    # Whether Runlight has made its tables. Until it has, an app's request starts init beside it, once at a
+    # time, and again no sooner than a minute after one that failed.
+    def ready?
+      return true unless core.respond_to?(:ready?)
+      return true if core.ready?
+
+      @lock.synchronize do
+        return false if @warming&.alive?
+        return false if @warm_failed_at && monotonic - @warm_failed_at < 60
+
+        @warming = Thread.new do
+          core.init
+        rescue StandardError => e
+          @warm_failed_at = monotonic
+          warn "Runlight: could not get ready #{e.message}"
+        end
+      end
+      false
+    end
+
+    # AI agent fetches are recorded once Runlight is ready, and never hold up or break the page.
+    def observe(request)
+      core.observe(request) if ready?
+    rescue StandardError => e
+      warn "Runlight: #{e.message}"
+    end
+
+    def monotonic
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
     def idle
       core.idle if core.respond_to?(:idle)
     rescue StandardError => e