runlight 0.1.0 → 0.1.3

data/lib/runlight/routes.rb

@@ -18,8 +18,14 @@ module Runlight
   # - cronSecret: also accepted as a bearer token on POST /api/check. Defaults to CRON_SECRET.
   # - observeKey: lets another site report AI agent fetches to POST /api/observe. Defaults to RUNLIGHT_OBSERVE_KEY.
   # - signOut, signIn: links the dashboard shows. The standalone server sets them.
-  # - accounts: true for sign-in accounts, or a Runlight::Accounts::Web of your own.
+  # - accounts: true for sign-in accounts, or a Runlight::Accounts::Web of your own. Sessions are signed with the
+  #   Core's secret (RUNLIGHT_SECRET, else RUNLIGHT_TOKEN). With the token left nil and no secret, only
+  #   development makes one up (a restart signs everyone out); anywhere else that raises ArgumentError, since
+  #   each process of an app would make its own and sign the others' sessions out.
   # - geoCredit: credits DB-IP in the dashboard's footer.
+  # - cms: the CMS that adds the script to every page, such as "WordPress", when Runlight runs inside one. The
+  #   dashboard then says so in place of the steps to add the script, and pages of the same origin, the CMS's own
+  #   admin, may show the whole dashboard in a frame. The WordPress, Drupal, and Craft plugins set it.
   # - origin: the address people open the app at, such as https://example.com.
   # - ownHosts: a callable returning more names the dashboard is reached at.
   # - accountOf: a callable taking the Request and returning the account it comes from, or nil (internal, for the
@@ -44,6 +50,14 @@ module Runlight
     PICK_TARGET_PLACEHOLDER = '"__RUNLIGHT_PICK_TARGET__"'
     # Where the hostnames of the site its ticket names go, as JSON inside a string.
     PICK_HOSTS_PLACEHOLDER = '"__RUNLIGHT_PICK_HOSTS__"'
+    # Where the relay page the picker sends its choice to goes, the return address its ticket names.
+    PICK_RETURN_PLACEHOLDER = '"__RUNLIGHT_PICK_RETURN__"'
+    # Where the nonce the dashboard made for this pick goes, which the relay page hands back to it.
+    PICK_NONCE_PLACEHOLDER = '"__RUNLIGHT_PICK_NONCE__"'
+    # The relay page on a dashboard's origin: the dashboard's address with /pick-return on the end.
+    PICK_RETURN = %r{\Ahttps?://[^/?#\s]+(/[^?#\s]*)?/pick-return\z}
+    # What the dashboard names one pick by, so its relay page answers only the tab that started it.
+    PICK_NONCE = /\A[a-f0-9]{32}\z/
     # How long a picker ticket works: long enough to find the element, not to be kept.
     PICK_TICKET_MS = 30 * 60_000
     # Questions one person may put to the assistant in an hour, and at once.
@@ -59,10 +73,14 @@ module Runlight
     PATH_DIMENSIONS = %w[page entry exit ai_page].freeze
     # runlight.ts's LINK_DOMAIN_CHECK: the path on every link domain that answers when the domain reaches this Runlight.
     LINK_DOMAIN_CHECK = "/.well-known/runlight-link-domain"
+    # The most read of a link domain's answer to the check, and of an install's live answer for a share or embed.
+    LINK_DOMAIN_MAX_BYTES = 16_384
+    REALTIME_MAX_BYTES = 4 * 1024 * 1024
     # runlight.ts's RETENTION_MONTHS: the choices for how long a site keeps its visits.
     RETENTION_MONTHS = [6, 12, 24, 36, 60].freeze
-    private_constant :RULES_PLACEHOLDER, :PICK_TARGET_PLACEHOLDER, :PICK_HOSTS_PLACEHOLDER, :SHARE_ID,
-                     :PATH_DIMENSIONS, :LINK_DOMAIN_CHECK, :RETENTION_MONTHS
+    private_constant :RULES_PLACEHOLDER, :PICK_TARGET_PLACEHOLDER, :PICK_HOSTS_PLACEHOLDER, :PICK_RETURN_PLACEHOLDER,
+                     :PICK_NONCE_PLACEHOLDER, :PICK_RETURN, :PICK_NONCE, :SHARE_ID,
+                     :PATH_DIMENSIONS, :LINK_DOMAIN_CHECK, :LINK_DOMAIN_MAX_BYTES, :REALTIME_MAX_BYTES, :RETENTION_MONTHS
 
     DASHBOARD_CSP = "default-src 'none'; script-src 'self'; style-src 'self'; img-src 'self' data:; connect-src 'self'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'"
 
@@ -73,9 +91,7 @@ module Runlight
     EMAIL = /\A[^#{Js::SPACE}@<>"]+@[^#{Js::SPACE}@<>"]+\.[^#{Js::SPACE}@<>"]+\z/
     # A dashboard's origin, which a picker ticket names.
     ORIGIN = %r{\Ahttps?://[^/?\##{Js::SPACE}]+\z}
-    # An address a report's links may point back to.
-    HOME = %r{\Ahttps?://[^#{Js::SPACE}]+\z}
-    private_constant :EMAIL, :ORIGIN, :HOME
+    private_constant :EMAIL, :ORIGIN
 
     # The files in assets/, read once per process.
     @assets = {}
@@ -118,11 +134,17 @@ module Runlight
 
       # Accounts: the standalone server passes its own, and an app turns them on with true. Sessions need a secret
       # that outlives the process; in development without one, a made-up one does, so a restart signs everyone out.
-      # An app left open on purpose (token: nil) is treated like development here.
+      # An app left open on purpose (token: nil) is treated like development for the first account, but outside
+      # development it still needs a real secret: TypeScript runs as one process, where a made-up one only lasts
+      # until a restart, but a Ruby app usually runs several, and each would sign the others' sessions out.
       token = @token
       open_setup = token.nil? || (token == "" && development?)
-      account_secret = runlight.secret || (open_setup ? Hashing.random_id(32) : nil)
       accounts = options["accounts"]
+      if accounts == true && token.nil? && runlight.secret.to_s.empty? && !development?
+        raise ArgumentError, "Runlight: accounts need a secret every process shares; set RUNLIGHT_SECRET (or pass secret:) to a long random string"
+      end
+
+      account_secret = runlight.secret || (open_setup ? Hashing.random_id(32) : nil)
       @web = if accounts.is_a?(Accounts::Web)
                accounts
              elsif accounts == true && !account_secret.nil? && account_secret != ""
@@ -236,17 +258,22 @@ module Runlight
         Js.scrub(value).match?(EMAIL)
       end
 
-      # A plain page in a visitor's language, for unsubscribing and for a share link that is gone.
-      def small_page(lang, body, status = 200)
+      # A plain page in a visitor's language, for unsubscribing, for a share link that is gone, and for the picker's
+      # relay page, which alone runs a script: one file from this server, never inline.
+      def small_page(lang, body, status = 200, script = "")
         Http::Response.new(
           "<!doctype html><html lang=\"#{lang}\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\"><meta name=\"robots\" content=\"noindex\"><title>Runlight</title>\n" \
           '<style>body{margin:0;min-height:100vh;display:grid;place-items:center;background:#f4f4f5;font:16px/1.5 -apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;color:#111827}main{max-width:420px;margin:24px;padding:32px;background:#fff;border:1px solid #e5e7eb;border-radius:14px}h1{font-size:20px;margin:0 0 12px}p{margin:0 0 20px;color:#4b5563}button{height:40px;padding:0 18px;border:0;border-radius:8px;background:#111827;color:#fff;font:inherit;font-weight:600;cursor:pointer}</style></head><body><main>' \
-          "#{body}</main></body></html>",
+          "#{body}</main>#{script == "" ? "" : "<script src=\"#{escape_html(script)}\"></script>"}</body></html>",
           status: status,
           headers: {
             "content-type" => "text/html; charset=utf-8",
             "cache-control" => "no-store",
-            "content-security-policy" => "default-src 'none'; style-src 'unsafe-inline'; form-action 'self'; frame-ancestors 'none'",
+            "content-security-policy" => if script == ""
+                                           "default-src 'none'; style-src 'unsafe-inline'; form-action 'self'; frame-ancestors 'none'"
+                                         else
+                                           "default-src 'none'; script-src 'self'; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'"
+                                         end,
             "referrer-policy" => "no-referrer",
           },
         )
@@ -366,7 +393,7 @@ module Runlight
       # The dashboard's page, which holds no data: the API it calls checks access. `embed` is the dashboard inside a
       # CMS's admin pages: { "session", "origin" }, its session (empty once its ticket was used or ran out) and the
       # admin origin that frames it.
-      def dashboard(base, share = "", sign_out = "", geo_credit = false, accounts = false, sign_in = "", embed = nil)
+      def dashboard(base, share = "", sign_out = "", geo_credit = false, accounts = false, sign_in = "", embed = nil, cms = "")
         b = escape_attr(base)
         hash = build_hash("dashboardHash")
         attributes = (share != "" ? " data-share=\"#{escape_attr(share)}\"" : "") +
@@ -374,6 +401,7 @@ module Runlight
                      (sign_in != "" ? " data-sign-in=\"#{escape_attr(sign_in)}\"" : "") +
                      (geo_credit ? ' data-geo-credit=""' : "") +
                      (accounts ? ' data-accounts=""' : "") +
+                     (cms != "" ? " data-cms=\"#{escape_attr(cms)}\"" : "") +
                      (embed.nil? ? "" : " data-embed=\"#{escape_attr(embed["session"])}\" data-embed-origin=\"#{escape_attr(embed["origin"])}\"")
         "<!doctype html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\">\n<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n<meta name=\"robots\" content=\"noindex\">\n<title>Runlight</title>\n" \
           "<link rel=\"icon\" href=\"#{Brand.runlight_icon}\">\n" \
@@ -406,12 +434,10 @@ module Runlight
         Js.scrub(value).match?(ORIGIN)
       end
 
-      # decodeURIComponent, which throws on a broken escape, as the TypeScript does (an internal error there).
-      def decode(text)
-        decoded = Js.decode_uri_component(text)
-        raise ArgumentError, "URI malformed" if decoded.nil?
-
-        decoded
+      # One part of a URL path with its percent escapes decoded, or nil when they are broken (such as %E0 or
+      # %zz). A route answers nil as it does a name it does not know, never with an error.
+      def decode_part(text)
+        Js.decode_uri_component(text)
       end
 
       # `String(body[key] ?? fallback)`.
@@ -504,7 +530,7 @@ module Runlight
     extend Helpers
     include Helpers
 
-    PUBLIC_HELPERS = %i[coded json json? host_name cookie_value read_cookie bearer normalise_base dashboard manage_path].freeze
+    PUBLIC_HELPERS = %i[coded json json? host_name cookie_value read_cookie bearer normalise_base dashboard manage_path oauth_document?].freeze
     private_constant :PUBLIC_HELPERS
     private(*Helpers.instance_methods(false))
     private_class_method(*(Helpers.instance_methods(false) - PUBLIC_HELPERS))
@@ -518,6 +544,8 @@ module Runlight
       if base != "" && oauth_document?(url.pathname)
         begin
           return OAuth.oauth_response(@oauth, request, url.pathname, url, context) || coded("Not found", "not_found", 404)
+        rescue TablesBusyError
+          return tables_busy
         rescue StandardError => e
           warn("Runlight: #{e.full_message(highlight: false)}")
           return coded("Internal error", "internal", 500)
@@ -530,13 +558,16 @@ module Runlight
 
       begin
         route(request, path, url, context)
+      rescue TablesBusyError
+        tables_busy
       rescue StandardError => e
         warn("Runlight: #{e.full_message(highlight: false)}")
         coded("Internal error", "internal", 500)
       end
     end
 
-    # The routes as a Rack app. The answer to a HEAD has no body.
+    # The routes as a Rack app. The answer to a HEAD has no body. Once the answer is sent, the work left for
+    # later (a retention change's deletions) runs in the Core's idle, as RackApp runs it.
     def call(env)
       begin
         request = Http::Request.from_rack(env)
@@ -545,11 +576,21 @@ module Runlight
                 [Json.encode({ "error" => "That request is too large" })]]
       end
       status, headers, body = handle(request).to_rack
-      [status, headers, request.method == "HEAD" ? [] : body]
+      idle = lambda do
+        @rl.idle
+      rescue StandardError => e
+        warn("Runlight: #{e.message}")
+      end
+      [status, headers, RackApp::Body.new(request.method == "HEAD" ? [] : body, idle)]
     end
 
     private
 
+    # Another process is building the tables, such as an index after an upgrade: worth asking again soon.
+    def tables_busy
+      coded("Runlight is creating or upgrading its tables. Try again in a minute.", "tables_busy", 503, nil, { "retry-after" => "30" })
+    end
+
     def sites
       @rl.sites
     end
@@ -672,7 +713,7 @@ module Runlight
 
     # Answers a read for a site counted by another install by asking that install,
     # with its token and its own id for the site, and handing back what it says.
-    def pass_through(remote, path, url, request = nil)
+    def pass_through(remote, path, url, request = nil, max_bytes = nil)
       target = Http::Url.new("#{remote["url"]}#{path}")
       query = target.search_params
       url.search_params.each { |key, value| query.append(key.to_s, value) }
@@ -694,6 +735,8 @@ module Runlight
           "timeoutMs" => write ? 30_000 : 120_000,
         }
         init["body"] = request.text if write
+        # One byte more than the most wanted is asked, so an answer past it is seen as such.
+        init.merge!("maxBytes" => max_bytes + 1, "truncate" => true) unless max_bytes.nil?
         answer = Safefetch.owner_fetch(target.href, init, @rl.fetcher, @rl.local_installs)
       rescue StandardError => e
         if e.is_a?(Http::FetchError) && e.timed_out?
@@ -764,6 +807,44 @@ module Runlight
       Http::Response.new(answer.text, status: answer.status, headers: back)
     end
 
+    # The checks a link domain passes before a site gets it, from Settings or from an import: the name as it
+    # is kept, or the refusal to send.
+    def check_link_domain(request, url, site, value)
+      domain = Js.lower(Js.trim(value))
+      domain = domain.sub(%r{\Ahttps?://}, "")
+      domain = domain.sub(%r{/.*\z}, "")
+      domain = domain.sub(/\.+\z/, "")
+      domain = domain.sub(/\Awww\./, "")
+      return coded("That is not a domain name", "domain_invalid", 400) unless domain.match?(DOMAIN_NAME)
+
+      if private_name?(domain) || Safefetch.resolves_privately?(domain)
+        return coded("#{domain} is not a public domain name. Use one that browsers anywhere can reach.", "domain_not_public", 400, { "domain" => domain })
+      end
+
+      # A link domain answers every path on it, so it must never be where the dashboard or a counted site lives.
+      # The request's own Host is the caller's to choose, so the configured address and the names people
+      # signed in from count too. A hub cannot know every name this app answers on, so it adds none until
+      # the app knows its own address.
+      return origin_needed if @managed.key?(request) && @origin.nil?
+
+      here = [request.headers.get("host"), request.headers.get("x-forwarded-host"), url.host].select { |h| Js.truthy?(h) }
+      own = [*(@origin.nil? ? [] : [Http::Url.new(@origin).host]), *here]
+      @options["ownHosts"]&.call&.each { |host| own << host.to_s }
+      taken = own.map { |h| host_name(h) }
+      sites.each do |s|
+        taken.concat(s["hostnames"])
+        taken.concat(@rl.remote(s["id"])&.[]("hostnames") || [])
+      end
+      if taken.include?(domain)
+        return coded("#{domain} is where this dashboard or one of your sites lives. Use a separate domain or subdomain for short links, such as go.#{domain}.", "domain_in_use", 400, { "domain" => domain })
+      end
+
+      owner = store.link_domains.find { |d| d["domain"] == domain }
+      return coded("#{domain} already belongs to another site", "domain_taken", 409, { "domain" => domain }) if !owner.nil? && owner["site"] != site
+
+      domain
+    end
+
     def links_api(request, path, url)
       @rl.init
       site = query_site(url)
@@ -778,49 +859,17 @@ module Runlight
             body = read_json(request)
             return body if body.is_a?(Http::Response)
 
-            domain = Js.lower(Js.trim(text(body, "domain")))
-            domain = domain.sub(%r{\Ahttps?://}, "")
-            domain = domain.sub(%r{/.*\z}, "")
-            domain = domain.sub(/\.+\z/, "")
-            domain = domain.sub(/\Awww\./, "")
-            return coded("That is not a domain name", "domain_invalid", 400) unless domain.match?(DOMAIN_NAME)
-
-            if private_name?(domain) || Safefetch.resolves_privately?(domain)
-              return coded("#{domain} is not a public domain name. Use one that browsers anywhere can reach.", "domain_not_public", 400, { "domain" => domain })
-            end
-
-            # A link domain answers every path on it, so it must never be where the dashboard or a counted site lives.
-            # The request's own Host is the caller's to choose, so the configured address and the names people
-            # signed in from count too. A hub cannot know every name this app answers on, so it adds none until
-            # the app knows its own address.
-            return origin_needed if @managed.key?(request) && @origin.nil?
-
-            here = [request.headers.get("host"), request.headers.get("x-forwarded-host"), url.host].select { |h| Js.truthy?(h) }
-            own = [*(@origin.nil? ? [] : [Http::Url.new(@origin).host]), *here]
-            @options["ownHosts"]&.call&.each { |host| own << host.to_s }
-            taken = own.map { |h| host_name(h) }
-            sites.each do |s|
-              taken.concat(s["hostnames"])
-              taken.concat(@rl.remote(s["id"])&.[]("hostnames") || [])
-            end
-            if taken.include?(domain)
-              return coded("#{domain} is where this dashboard or one of your sites lives. Use a separate domain or subdomain for short links, such as go.#{domain}.", "domain_in_use", 400, { "domain" => domain })
-            end
+            domain = check_link_domain(request, url, site["id"], text(body, "domain"))
+            return domain if domain.is_a?(Http::Response)
 
-            store.link_domains.each do |d|
-              next unless d["domain"] == domain
-              return coded("#{domain} already belongs to another site", "domain_taken", 409, { "domain" => domain }) if d["site"] != site["id"]
-
-              break
-            end
             store.add_link_domain(domain, site["id"], @rl.now)
             @rl.forget_link_domains
             return json({ "domain" => domain }, 201)
           end
         end
         if (check_match = path.match(%r{\A/api/link-domains/([^/]+)/check\z})) && request.method == "GET"
-          domain = decode(check_match[1])
-          return coded("Unknown domain", "unknown_domain", 404) unless own_domains.call.include?(domain)
+          domain = decode_part(check_match[1])
+          return coded("Unknown domain", "unknown_domain", 404) if domain.nil? || !own_domains.call.include?(domain)
 
           # One added before names inside private networks were refused is never fetched.
           # What the check found, as a code the dashboard says in its own words, beside the English reason.
@@ -841,9 +890,11 @@ module Runlight
           begin
             # Only a public address is fetched, whatever the name resolves to now, so the check cannot be pointed
             # into a private network.
-            answer = Safefetch.public_fetch("https://#{domain}#{LINK_DOMAIN_CHECK}", { "timeoutMs" => 5000 }, @rl.fetcher)
+            # Read no further than the check's answer could need; one byte more is asked, so a longer one is seen as such.
+            answer = Safefetch.public_fetch("https://#{domain}#{LINK_DOMAIN_CHECK}",
+                                            { "timeoutMs" => 5000, "maxBytes" => LINK_DOMAIN_MAX_BYTES + 1, "truncate" => true }, @rl.fetcher)
             parsed, body = begin
-              Js.parse_json(answer.text)
+              Js.parse_json(Body.read_text_capped(answer, LINK_DOMAIN_MAX_BYTES))
             rescue StandardError
               [false, nil]
             end
@@ -860,8 +911,8 @@ module Runlight
         end
 
         if (domain_match = path.match(%r{\A/api/link-domains/([^/]+)\z})) && request.method == "DELETE"
-          domain = decode(domain_match[1])
-          return coded("Unknown domain", "unknown_domain", 404) unless own_domains.call.include?(domain)
+          domain = decode_part(domain_match[1])
+          return coded("Unknown domain", "unknown_domain", 404) if domain.nil? || !own_domains.call.include?(domain)
 
           store.remove_link_domain(domain)
           @rl.forget_link_domains
@@ -903,6 +954,11 @@ module Runlight
               credentials(Js.get(body, "credentials")),
               cursor.is_a?(String) ? cursor : nil,
               done.is_a?(Float) && done.nan? ? 0 : done,
+              # An imported link's domain passes the same checks as one added in Settings, or the link comes over without it.
+              lambda do |domain|
+                checked = check_link_domain(request, url, site["id"], domain)
+                checked.is_a?(Http::Response) ? nil : checked
+              end,
             )
             return json(step)
           rescue Importers::ImportError => e
@@ -976,40 +1032,41 @@ module Runlight
     # The key picker tickets are signed with, made on first use and kept in the database for every process.
     def pick_key
       @rl.init
-      saved = store.setting("pick-key")
-      return saved if !saved.nil? && saved != ""
-
-      made = Hashing.random_id(32)
-      store.set_setting("pick-key", made)
-      made
+      # Written only if no other request wrote one first, so nothing signed with a key that lost goes stale.
+      store.setting_or_add("pick-key") { Hashing.random_id(32) }
     end
 
-    # A ticket that lets the picker, on `site`'s pages, send its choice to `origin`, the dashboard that asked, for half an hour.
-    def pick_ticket(origin, site)
+    # A ticket that lets the picker, on `site`'s pages, send its choice to `origin`, the dashboard that asked, for half
+    # an hour. With a return address and a nonce it sends it there, to the relay page, which hands it to the dashboard
+    # tab that made the nonce; without them, as an older dashboard asks, to the tab that opened the site.
+    def pick_ticket(origin, site, back = nil)
       payload = "#{@rl.now + PICK_TICKET_MS}.#{site.unpack1("H*")}.#{origin.unpack1("H*")}"
+      payload += ".#{back["url"].unpack1("H*")}.#{back["nonce"]}" unless back.nil?
       "#{payload}.#{Hashing.hmac(pick_key, payload)}"
     end
 
-    # The dashboard origin and site a picker ticket names, or nil when it is not one this install signed or has run out.
+    # What a picker ticket names, or nil when it is not one this install signed or has run out.
     def pick_target(ticket)
-      parts = ticket.match(/\A(\d+)\.([a-f0-9]{2,512})\.([a-f0-9]{2,512})\.([a-f0-9]{64})\z/)
+      parts = ticket.match(/\A(\d+)\.([a-f0-9]{2,512})\.([a-f0-9]{2,512})(?:\.([a-f0-9]{2,2048})\.([a-f0-9]{32}))?\.([a-f0-9]{64})\z/)
       return nil if parts.nil? || Js.number(parts[1]) < @rl.now
-      return nil unless constant_time_equal(parts[4], Hashing.hmac(pick_key, "#{parts[1]}.#{parts[2]}.#{parts[3]}"))
+      return nil unless constant_time_equal(parts[6], Hashing.hmac(pick_key, ticket[0, ticket.rindex(".")]))
 
       unhex = ->(text) { Js.scrub([text[0, text.length - (text.length % 2)]].pack("H*")) }
       origin = unhex.call(parts[3])
-      origin?(origin) ? { "origin" => origin, "site" => unhex.call(parts[2]) } : nil
+      return nil unless origin?(origin)
+      return { "origin" => origin, "site" => unhex.call(parts[2]) } if parts[4].nil?
+
+      address = unhex.call(parts[4])
+      return nil unless address.match?(PICK_RETURN) && address.start_with?("#{origin}/")
+
+      { "origin" => origin, "site" => unhex.call(parts[2]), "back" => { "url" => address, "nonce" => parts[5] } }
     end
 
     # The key embed tickets and sessions are signed with, made on first use and kept in the database for every process.
     def embed_key
       @rl.init
-      saved = store.setting("embed-key")
-      return saved if !saved.nil? && saved != ""
-
-      made = Hashing.random_id(32)
-      store.set_setting("embed-key", made)
-      made
+      # Written only if no other request wrote one first, so nothing signed with a key that lost goes stale.
+      store.setting_or_add("embed-key") { Hashing.random_id(32) }
     end
 
     # An embed token that still exists, for a site that still does.
@@ -1096,10 +1153,10 @@ module Runlight
       return site if site.is_a?(Http::Response)
 
       existing = store.goals(site["id"])
-      id = path == "/api/goals" ? nil : decode(path["/api/goals/".length..])
+      id = path == "/api/goals" ? nil : decode_part(path["/api/goals/".length..])
       before = nil
       existing.each { |g| before = g if g["id"] == id }
-      return coded("Unknown goal", "unknown_goal", 404) if !id.nil? && before.nil?
+      return coded("Unknown goal", "unknown_goal", 404) if path != "/api/goals" && (id.nil? || before.nil?)
 
       @trackers = {}
       if request.method == "DELETE"
@@ -1125,7 +1182,7 @@ module Runlight
           if request.method == "GET"
             settings = @rl.mail_settings
             known = settings || {}
-            service = Mail::Transports::SERVICES.find { |s| s["id"] == known["service"] }
+            service = known["source"] == "app" ? nil : Mail::Transports::SERVICES.find { |s| s["id"] == known["service"] }
             # Secret fields come back only as "saved", never as their value.
             fields = {}
             saved = []
@@ -1139,16 +1196,23 @@ module Runlight
             end
             # A hub with a manage token learns which service sends the reports and from where, nothing more.
             via_manage = @managed.key?(request)
-            return json({
+            answer = {
               "source" => known["source"],
-              "service" => known["service"] || "",
+              "service" => known["source"] == "app" ? "" : known["service"] || "",
               "from" => known["from"] || "",
               "fromName" => known["fromName"] || "",
               "fields" => via_manage ? {} : fields,
               "saved" => via_manage ? [] : saved,
               "encrypted" => !@rl.secret.nil?,
-              "services" => Mail::Transports::SERVICES,
-            })
+              "services" => @rl.mail_services,
+            }
+            # The app's own mailer, named only when there is one, so the answer without it stays as it was.
+            app = @rl.app_mailer
+            unless app.nil?
+              answer["app"] = { "via" => app["via"], "from" => app["from"], "fromName" => app["fromName"] || "",
+                                "replaced" => known["source"] != "app" }
+            end
+            return json(answer)
           end
           if request.method == "PUT"
             body = read_json(request)
@@ -1175,7 +1239,11 @@ module Runlight
           return coded("Set up a mail service first", "mail_unset", 400) if settings.nil?
 
           t = Messages.translator(text(body, "lang", "en"))["t"]
-          name = Mail::Transports::SERVICES.find { |s| s["id"] == settings["service"] }&.[]("name") || ""
+          name = if settings["source"] == "app"
+                   settings["via"] || ""
+                 else
+                   Mail::Transports::SERVICES.find { |s| s["id"] == settings["service"] }&.[]("name") || ""
+                 end
           @rl.send_mail({
             "to" => to,
             "subject" => t.call("email.test.subject"),
@@ -1208,13 +1276,13 @@ module Runlight
             end
             return coded("A site can send to at most 50 addresses", "report_limit", 400) if existing.length >= 50
 
-            # Links in the email point back to the configured address, or else to this dashboard as the
-            # browser sees it. A report made from a hub needs the configured address, where its unsubscribe
-            # link answers, since the Host its request names is the hub's to choose.
+            # Links in the email point back to the configured address, or else to this dashboard at the address
+            # the request came to. An address in the body is never used, so whoever adds a report cannot point
+            # its links somewhere else. A report made from a hub needs the configured address, where its
+            # unsubscribe link answers, since the Host its request names is the hub's to choose.
             return origin_needed if @managed.key?(request) && @origin.nil?
 
-            given = @origin.nil? ? text(body, "origin") : ""
-            home = Js.scrub(given).match?(HOME) ? given.sub(%r{/+\z}, "") : "#{@origin || url.origin}#{@base}"
+            home = "#{@origin || url.origin}#{@base}"
             # A period already due counts as sent, so a report added mid-week first goes out on the next Monday, as the form says.
             now = @rl.now
             due = Reports.last_period(frequency, now, site["timezone"])
@@ -1308,8 +1376,8 @@ module Runlight
         return coded("Method not allowed", "method_not_allowed", 405)
       end
 
-      id = decode(path["/api/shares/".length..])
-      share = id.match?(SHARE_ID) ? store.share_by_id(id) : nil
+      id = decode_part(path["/api/shares/".length..])
+      share = !id.nil? && id.match?(SHARE_ID) ? store.share_by_id(id) : nil
       return coded("Unknown share", "unknown_share", 404) if share.nil? || share["site"] != site["id"]
 
       if request.method == "PATCH"
@@ -1338,39 +1406,72 @@ module Runlight
     # As in TypeScript, the hour's questions are counted in this process; the day's, in the settings.
     def ask_turn(who, owner)
       now = @rl.now
+      # Taken in one step, before anything slow, so questions sent at once each see the others.
       mine = @lock.synchronize do
-        held = @asked[who] || { "at" => [], "open" => 0 }
+        held = (@asked[who] ||= { "at" => [], "open" => 0 })
         held["at"] = held["at"].select { |at| now - at < 3_600_000 }
-        held
-      end
-      if mine["at"].length >= ASK_PER_HOUR || mine["open"] >= ASK_AT_ONCE
-        return coded("You have asked a lot in a short time. Wait a little and ask again.", "assistant_soon", 429)
+        if held["at"].length >= ASK_PER_HOUR || held["open"] >= ASK_AT_ONCE
+          nil
+        else
+          held["at"] << now
+          held["open"] += 1
+          held
+        end
       end
+      return coded("You have asked a lot in a short time. Wait a little and ask again.", "assistant_soon", 429) if mine.nil?
 
+      release = -> { @lock.synchronize { mine["open"] -= 1 } }
       unless owner
-        daily = viewer_daily
-        day = "assistant-asked:#{Time.at(now.div(1000)).utc.strftime("%Y-%m-%d")}"
-        counts = Json.decode(store.setting(day) || "{}")
-        if (counts[who] || 0) >= daily
-          return coded("Viewers can ask #{Js.string(daily)} questions a day. Ask again tomorrow.", "assistant_daily", 429, { "limit" => Js.string(daily) })
-        end
+        counted = begin
+          count_ask(who, now)
+        rescue StandardError
+          release.call
+          raise
+        end
+        unless counted == true
+          release.call
+          @lock.synchronize do
+            at = mine["at"].index(now)
+            mine["at"].delete_at(at) unless at.nil?
+          end
+          return coded("You have asked a lot in a short time. Wait a little and ask again.", "assistant_soon", 429) if counted.nil?
 
-        counts[who] = (counts[who] || 0) + 1
-        store.set_setting(day, Json.encode(counts))
-        store.settings_starting_with("assistant-asked:").each do |entry|
-          store.set_setting(entry["key"], nil) if entry["key"] != day
+          return coded("Viewers can ask #{Js.string(counted)} questions a day. Ask again tomorrow.", "assistant_daily", 429, { "limit" => Js.string(counted) })
         end
       end
+      # People who stopped asking are dropped, so the map holds only the last hour's.
       @lock.synchronize do
-        mine["at"] << now
-        mine["open"] += 1
-        @asked[who] = mine
-        # People who stopped asking are dropped, so the map holds only the last hour's.
         if @asked.size > 1000
           @asked.delete_if { |_key, value| value["open"].zero? && value["at"].none? { |at| now - at < 3_600_000 } }
         end
       end
-      -> { @lock.synchronize { mine["open"] -= 1 } }
+      release
+    end
+
+    # Counts a viewer's question in their own row for the day, or gives the daily limit when it is reached. The
+    # row only ever moves from the count read to the next one, so of questions sent at once, by one viewer or by
+    # every process on the database, each count is taken once. True when counted, nil when the row kept changing.
+    def count_ask(who, now)
+      limit = viewer_daily
+      return limit if limit < 1
+
+      day = "assistant-asked:#{Time.at(now.div(1000)).utc.strftime("%Y-%m-%d")}:"
+      key = "#{day}#{who}"
+      20.times do
+        seen = store.setting(key)
+        if seen.nil?
+          next unless store.add_setting(key, "1")
+
+          # A viewer's first question of the day clears the days before.
+          store.settings_starting_with("assistant-asked:").each do |entry|
+            store.set_setting(entry["key"], nil) unless entry["key"].start_with?(day)
+          end
+          return true
+        end
+        return limit if Js.number(seen) >= limit
+        return true if store.swap_setting(key, seen, Js.string(Js.number(seen) + 1))
+      end
+      nil
     end
 
     def tokens_api(request, path)
@@ -1394,7 +1495,7 @@ module Runlight
         asked_scope = Js.get(body, "scope")
         scope = %w[manage embed].include?(asked_scope) ? asked_scope : "read"
         return coded("A token that changes settings is for one site. Pick the site.", "token_site", 400) if scope == "manage" && site == ""
-        return coded("A key for the dashboard in a CMS is for one site. Pick the site.", "embed_site", 400) if scope == "embed" && site == ""
+        return coded("A dashboard key is for one site. Pick the site.", "embed_site", 400) if scope == "embed" && site == ""
 
         secret = "#{TOKEN_PREFIX}#{Hashing.random_id(20)}"
         row = { "id" => Hashing.random_id, "name" => name, "site" => site, "scope" => scope, "hash" => Hashing.sha256(secret),
@@ -1508,7 +1609,7 @@ module Runlight
       if path == "/api/embed" && method == "POST"
         token = api_token(request)
         return denied(false) if token.nil?
-        return coded("Use a key for the dashboard in a CMS, made in Settings, Install", "embed_token", 403) if token["scope"] != "embed"
+        return coded("Use the site's dashboard key, made in Settings, Install", "embed_token", 403) if token["scope"] != "embed"
 
         site = token["site"] != "" ? rl.site(token["site"]) : nil
         return coded("Unknown site", "unknown_site", 404) if site.nil?
@@ -1520,7 +1621,15 @@ module Runlight
         parsed = origin?(origin) && Js.length(origin) <= 200 ? Http::Url.parse(origin) : nil
         return coded("Send the admin page's origin, such as https://example.com", "embed_origin", 400) if parsed.nil? || parsed.origin != origin
 
-        host = host_name(parsed.host)
+        # Only https and the default port, so a plain http page or another service on the same name cannot frame
+        # the dashboard; a page on this computer, for development, may use http and any port. The name must be
+        # one of the site's own, which are kept without www, so www. is the only difference allowed.
+        local = ["localhost", "127.0.0.1", "[::1]"].include?(parsed.hostname)
+        if !local && (parsed.protocol != "https:" || parsed.port != "")
+          return coded("Send the admin page's origin, such as https://example.com", "embed_origin", 400)
+        end
+
+        host = parsed.hostname.sub(/\Awww\./, "")
         domains = ((rl.remote(site["id"]) || {})["hostnames"] || site["hostnames"] || []).map { |h| host_name(h) }
         unless domains.include?(host)
           return coded("#{host} is not one of this site's domains. Add it to the site's domains in Runlight's settings.", "embed_host", 400, { "host" => host })
@@ -1536,7 +1645,7 @@ module Runlight
       if !token.nil? && token["scope"] == "manage" && manage_path(method, path)
         asked = url.search_params.get("site")
         site_match = path.match(%r{\A/api/sites/([^/]+)\z})
-        if (!asked.nil? && asked != "" && asked != token["site"]) || (!site_match.nil? && decode(site_match[1]) != token["site"])
+        if (!asked.nil? && asked != "" && asked != token["site"]) || (!site_match.nil? && decode_part(site_match[1]) != token["site"])
           return coded("Unknown site", "unknown_site", 404)
         end
 
@@ -1621,13 +1730,13 @@ module Runlight
         return site if site.is_a?(Http::Response)
 
         begin
-          return json(Importers::Visits.import_csv_visits(rl, site["id"], plain(Js.get(body, "rows"))))
+          return json(Importers::Visits.import_csv_visits(rl, site["id"], plain(Js.get(body, "rows")), Js.get(body, "secret")))
         rescue Importers::ImportError => e
           return refused(e, "import_failed")
         end
       end
 
-      # Each site's key for CMS plugins reporting AI agent fetches: made on first ask, replaced on request.
+      # Each site's agents key, for CMS plugins reporting AI agent fetches: made on first ask, replaced on request.
       if (path == "/api/observe-key" && method == "GET") || (path == "/api/observe-key/new" && method == "POST")
         access = can_read(request)
         return denied(access) if access != true
@@ -1636,13 +1745,7 @@ module Runlight
         site = query_site(url)
         return site if site.is_a?(Http::Response)
 
-        name = "observe-key:#{site["id"]}"
-        key = path.end_with?("/new") ? nil : store.setting(name)
-        if key.nil? || key == ""
-          key = "rlo_#{Hashing.random_id(20)}"
-          store.set_setting(name, key)
-        end
-        return json({ "key" => key })
+        return json({ "key" => OAuth.agents_key(rl, site["id"], path.end_with?("/new")) })
       end
 
       # Making, changing, and deleting funnels; reading them is with the other reports.
@@ -1826,13 +1929,25 @@ module Runlight
         origin = text(body, "origin")
         return coded("Send the dashboard's origin, such as https://stats.example.com", "pick_origin", 400) unless origin?(origin)
 
+        # Where the picker sends its choice: the relay page on the same origin, and the nonce the dashboard tab
+        # listens for. An older dashboard sends neither and gets a ticket for the tab that opened the site.
+        back = nil
+        if given?(body, "return") || given?(body, "nonce")
+          address = Js.scrub(text(body, "return"))
+          nonce = text(body, "nonce")
+          if address.unpack1("H*").length > 2048 || !address.match?(PICK_RETURN) || !address.start_with?("#{origin}/") || !nonce.match?(PICK_NONCE)
+            return coded("Send the dashboard's address with /pick-return on the end, on its origin, and a nonce of 32 hex digits", "pick_return", 400)
+          end
+
+          back = { "url" => address, "nonce" => nonce }
+        end
         # A hub's ticket only ever sends to the hub it connected from, never to an origin it names now.
         hub = @managed[request]
         if !hub.nil? && store.setting("token-origin:#{hub["id"]}") != origin
           return coded("This hub's address is not the one it connected from. Connect the site again from here.", "pick_hub", 403)
         end
 
-        return json({ "ticket" => pick_ticket(origin, site["id"]) })
+        return json({ "ticket" => pick_ticket(origin, site["id"], back) })
       end
 
       if (path == "/api/goals" && method == "POST") || (path.match?(%r{\A/api/goals/[^/]+\z}) && %w[PATCH DELETE].include?(method))
@@ -1869,8 +1984,11 @@ module Runlight
         access = can_read(request)
         return denied(access) if access != true
 
+        id = decode_part(site_match[1])
+        return coded("Unknown site", "unknown_site", 404) if id.nil?
+
         begin
-          rl.delete_site(decode(site_match[1]))
+          rl.delete_site(id)
           return json({ "ok" => true })
         rescue RangeError => e
           return e.message == "Unknown site" ? coded(e.message, "unknown_site", 404) : refused(e, "site_invalid")
@@ -2036,8 +2154,10 @@ module Runlight
         months = RETENTION_MONTHS.join(", ")
         return coded("Keep visits for #{months} months, or forever", "retention_bad", 400, { "months" => months })
       end
+      id = decode_part(raw_id)
+      return coded("Unknown site", "unknown_site", 404) if id.nil?
+
       begin
-        id = decode(raw_id)
         remote = rl.remote(id)
         # How long a connected site keeps visits, and the timezone its days follow, are the install's
         # settings: this server passes them on, and changes its own row only once the install took them.
@@ -2058,14 +2178,14 @@ module Runlight
           return answer unless answer.ok?
 
           rl.forget_remote_info(id)
-        elsif remote.nil? && !retention.equal?(UNDEFINED)
-          rl.set_retention(id, retention.nil? ? nil : Js.number(retention))
         end
+        # update_site checks every field, hostnames too, before it changes anything.
         patch = {}
+        patch["retentionMonths"] = retention.nil? ? nil : Js.number(retention) if remote.nil? && !retention.equal?(UNDEFINED)
         patch["name"] = Js.string(Js.get(body, "name")) if given?(body, "name")
         patch["timezone"] = Js.string(Js.get(body, "timezone")) if given?(body, "timezone")
         patch["hostnames"] = plain(Js.get(body, "hostnames")) if given?(body, "hostnames") && rl.managed_sites
-        site = rl.update_site(decode(raw_id), patch).dup
+        site = rl.update_site(id, patch).dup
         # A connected site answers as the list shows it, so the dashboard keeps its install and domains.
         unless remote.nil?
           site["remote"] = remote["url"]
@@ -2153,7 +2273,24 @@ module Runlight
       return coded("Unknown site", "unknown_site", 404) if site.nil? || (!only.nil? && site["id"] != only)
 
       remote = rl.remote(site["id"])
-      return pass_through(remote, path, url, request) unless remote.nil?
+      # A share link or an embedded dashboard sees how many are here now and what they read, but not the feed of
+      # each visit, with its time and city, since anyone with the link could watch one person's way through the site.
+      unfeed = ->(live) { shared.nil? ? live : live.merge("recent" => []) }
+      unless remote.nil?
+        live_for_share = path == "/api/realtime" && !shared.nil?
+        answer = pass_through(remote, path, url, request, live_for_share ? REALTIME_MAX_BYTES : nil)
+        return answer if !live_for_share || !answer.ok?
+
+        parsed, live = begin
+          Js.parse_json(Body.read_text_capped(answer, REALTIME_MAX_BYTES))
+        rescue Http::BodyTooLong
+          [false, nil]
+        end
+        return json(unfeed.call(live)) if parsed && live.is_a?(Hash)
+
+        host = Http::Url.new(remote["url"].to_s).host
+        return coded("Could not reach #{host}", "unreachable", 502, { "host" => host })
+      end
 
       if path == "/api/icon"
         host = site["hostnames"][0]
@@ -2170,7 +2307,7 @@ module Runlight
         })
       end
 
-      return json(store.realtime(site["id"], rl.now)) if path == "/api/realtime"
+      return json(unfeed.call(store.realtime(site["id"], rl.now))) if path == "/api/realtime"
 
       read = read_query(url, site)
       return read if read.is_a?(Http::Response)
@@ -2284,10 +2421,12 @@ module Runlight
 
         keys = store.event_prop_keys(query, event)
         asked = params.get("key")
-        # A property name goes into a JSON path on SQLite, so quotes and backslashes are refused.
-        return coded("Bad property name", "property_bad", 400) if !asked.nil? && !asked.match?(/\A[^"\\]{1,64}\z/)
+        # A property name goes into a JSON path on SQLite, so quotes and backslashes are refused, and without
+        # one asked for, the most used name that has none is shown.
+        safe = ->(name) { name.match?(/\A[^"\\]{1,64}\z/) }
+        return coded("Bad property name", "property_bad", 400) if !asked.nil? && !safe.call(asked)
 
-        key = asked || keys.first&.[]("key")
+        key = asked || keys.find { |k| safe.call(k["key"].to_s) }&.[]("key")
         count = limit(params.get("limit"), 100)
         rows = !key.nil? && key != "" ? store.event_prop_values(query, event, key, whole(count)) : []
         return json({ "site" => site["id"], "range" => range_out, "event" => event, "keys" => keys, "key" => key, "rows" => rows })
@@ -2373,11 +2512,32 @@ module Runlight
         target = pick_target(url.search_params.get("runlight_ticket") || "")
         rl.init unless target.nil?
         hosts = target.nil? ? [] : rl.site(target["site"])&.[]("hostnames")
-        script = replace_once(asset("picker.js"), PICK_TARGET_PLACEHOLDER, Json.encode(hosts.nil? || target.nil? ? "" : target["origin"]))
+        named = hosts.nil? ? nil : target
+        back = named&.[]("back")
+        script = replace_once(asset("picker.js"), PICK_TARGET_PLACEHOLDER, Json.encode(named.nil? ? "" : named["origin"]))
         script = replace_once(script, PICK_HOSTS_PLACEHOLDER, Json.encode(Json.encode(hosts || [])))
+        script = replace_once(script, PICK_RETURN_PLACEHOLDER, Json.encode(back.nil? ? "" : back["url"]))
+        script = replace_once(script, PICK_NONCE_PLACEHOLDER, Json.encode(back.nil? ? "" : back["nonce"]))
         return Http::Response.new(script, status: 200, headers: { "content-type" => "application/javascript; charset=utf-8", "cache-control" => "no-store" })
       end
 
+      # The relay page the picker opens with its choice in the fragment, which never reaches a server. Its
+      # script hands the choice to the dashboard tab on this origin that started the pick, and nothing else.
+      if path == "/pick-return" && method == "GET"
+        translator = Messages.translator(accepted_language(request))
+        t = translator["t"]
+        return small_page(
+          translator["lang"],
+          "<div id=\"done\" hidden><h1>#{escape_html(t.call("pick.doneTitle"))}</h1><p>#{escape_html(t.call("pick.done"))}</p></div>" \
+          "<div id=\"gone\" hidden><h1>#{escape_html(t.call("pick.goneTitle"))}</h1><p>#{escape_html(t.call("pick.gone"))}</p></div>",
+          200,
+          "#{base}/pick-return.js",
+        )
+      end
+      if path == "/pick-return.js" && method == "GET"
+        return Http::Response.new(asset("pick-return.js"), status: 200, headers: { "content-type" => "application/javascript; charset=utf-8", "cache-control" => "no-store" })
+      end
+
       if path == "/assets/world.#{build_hash("worldHash")}.json" && method == "GET"
         return Http::Response.new(asset("world.json"), status: 200, headers: { "content-type" => "application/json; charset=utf-8", "cache-control" => "public, max-age=31536000, immutable" })
       end
@@ -2499,11 +2659,13 @@ module Runlight
         end
         # The page itself holds no data; the API it calls checks access and
         # the page explains how to sign in when it is refused.
-        return Http::Response.new(dashboard(base, "", @sign_out || "", Js.truthy?(@options["geoCredit"]), !@web.nil?, @sign_in || ""), status: 200, headers: {
+        # Inside a CMS, its admin, on this same origin, shows the whole dashboard in a frame; nothing else may.
+        cms = @options["cms"].to_s
+        return Http::Response.new(dashboard(base, "", @sign_out || "", Js.truthy?(@options["geoCredit"]), !@web.nil?, @sign_in || "", nil, cms), status: 200, headers: {
           "content-type" => "text/html; charset=utf-8",
           "cache-control" => "no-store",
-          "content-security-policy" => DASHBOARD_CSP,
-          "x-frame-options" => "DENY",
+          "content-security-policy" => cms != "" ? DASHBOARD_CSP.sub("frame-ancestors 'none'", "frame-ancestors 'self'") : DASHBOARD_CSP,
+          "x-frame-options" => cms != "" ? "SAMEORIGIN" : "DENY",
           "referrer-policy" => "same-origin",
         })
       end