runlight 0.3.3 → 0.4.0

data/lib/runlight/agent_check.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+module Runlight
+  # Checks a request that names an AI agent against the addresses that agent's company publishes. Every
+  # list is in the same shape, {"prefixes": [{"ipv4Prefix": "a.b.c.d/n"} or {"ipv6Prefix": "x::/n"}]}.
+  #
+  # The answer is "verified" when the address is in the list, "mismatch" when the company publishes a list
+  # and the address is not in it, and "" when there is nothing to check against: no list for that agent,
+  # or no usable address. The address is only compared, never kept. The port of the SDK's agentcheck.ts.
+  module AgentCheck
+    # A list of more prefixes than this, or a reply bigger than MAX_LIST_BYTES, is not a list we read.
+    MAX_PREFIXES = 20_000
+    MAX_LIST_BYTES = 4_000_000
+    USER_AGENT = "Runlight (+https://runlight.sh)"
+
+    # Lists compiled to prefixes, by the list itself, as a WeakMap holds them in TypeScript. A list is
+    # compiled once while it is in use; the cache is emptied when it grows past what any install holds.
+    @compiled = {}.compare_by_identity
+    @compiled_lock = Mutex.new
+
+    module_function
+
+    # The address lists that came with this version: { "fetchedAt" => "YYYY-MM-DD", "lists" => { url => [prefix] } },
+    # read once from assets/agent-ranges.json.
+    def bundled
+      @bundled ||= begin
+        data = Json.decode(File.read(File.join(Version::ASSETS, "agent-ranges.json")))
+        lists = data["lists"].to_h { |url, list| [url.freeze, list.map { |p| p.dup.freeze }.freeze] }.freeze
+        { "fetchedAt" => data["fetchedAt"], "lists" => lists }.freeze
+      end
+    end
+
+    def ipv4(text)
+      parts = text.split(".", -1)
+      return nil unless parts.length == 4
+
+      n = 0
+      parts.each do |part|
+        return nil unless part.match?(/\A[0-9]{1,3}\z/)
+
+        byte = part.to_i
+        return nil if byte > 255
+
+        n = (n * 256) + byte
+      end
+      n
+    end
+
+    def ipv6(text)
+      return nil if !text.match?(/\A[0-9a-f:.]+\z/i) || text.split("::", -1).length > 2
+
+      # A dotted IPv4 tail ("::ffff:1.2.3.4") stands for the last two groups.
+      groups = text
+      if text.include?(".")
+        dot = text.rindex(":")
+        tail = ipv4(text[(dot + 1)..])
+        return nil if tail.nil?
+
+        groups = "#{text[0, dot + 1]}#{(tail >> 16).to_s(16)}:#{(tail & 0xffff).to_s(16)}"
+      end
+      halves = groups.split("::", -1)
+      head = halves[0]
+      rest = halves[1]
+      left = head.empty? ? [] : head.split(":", -1)
+      right = rest.nil? || rest.empty? ? [] : rest.split(":", -1)
+      missing = 8 - left.length - right.length
+      return nil if rest.nil? ? missing != 0 : missing < 1
+
+      n = 0
+      [*left, *Array.new(rest.nil? ? 0 : missing, "0"), *right].each do |group|
+        return nil unless group.match?(/\A[0-9a-f]{1,4}\z/i)
+
+        n = (n << 16) | group.to_i(16)
+      end
+      n
+    end
+
+    # An address as the check reads it, as [4 or 6, number]: surrounding space, [brackets], and an IPv6 zone
+    # ("%eth0") are ignored, and an IPv4 address written as IPv6 (::ffff:1.2.3.4) is read as IPv4. Nil when
+    # it is not one.
+    def parse_address(text)
+      s = Js.trim(text.to_s)
+      s = s[1...-1] if s.start_with?("[") && s.end_with?("]") && s.length >= 2
+      zone = s.index("%")
+      s = s[0, zone] unless zone.nil?
+      return nil if s.empty?
+
+      unless s.include?(":")
+        n = ipv4(s)
+        return n.nil? ? nil : [4, n]
+      end
+      n = ipv6(s)
+      return nil if n.nil?
+      return [4, n & 0xffffffff] if n >> 32 == 0xffff
+
+      [6, n]
+    end
+
+    # A prefix such as 20.171.206.0/24 or 2600:1f28::/56, as [4 or 6, base, bits], or nil. Bits past the length
+    # are ignored.
+    def parse_prefix(text)
+      slash = text.index("/")
+      return nil if slash.nil?
+
+      bits_text = text[(slash + 1)..]
+      return nil unless bits_text.match?(/\A[0-9]{1,3}\z/)
+
+      bits = bits_text.to_i
+      head = text[0, slash]
+      if head.include?(":")
+        n = ipv6(head)
+        return nil if n.nil? || bits > 128
+
+        return [6, n, bits]
+      end
+      n = ipv4(head)
+      return nil if n.nil? || bits > 32
+
+      [4, n, bits]
+    end
+
+    def covers?(prefix, address)
+      return false if prefix[0] != address[0]
+
+      shift = (prefix[0] == 4 ? 32 : 128) - prefix[2]
+      address[1] >> shift == prefix[1] >> shift
+    end
+
+    # The prefixes in a published list (decoded JSON), or nil when it is not a list in the expected shape (not
+    # an object, no prefixes, an entry that is not a prefix, or too many). One bad entry rejects the whole
+    # list, so a half broken download never replaces a good copy.
+    def read_range_list(body)
+      return nil unless body.is_a?(Hash)
+
+      prefixes = body["prefixes"]
+      return nil if !prefixes.is_a?(Array) || prefixes.empty? || prefixes.length > MAX_PREFIXES
+
+      prefixes.map do |entry|
+        return nil unless entry.is_a?(Hash)
+
+        v4 = entry["ipv4Prefix"]
+        v6 = entry["ipv6Prefix"]
+        text = if v4.is_a?(String) then v4
+               elsif v6.is_a?(String) then v6
+               end
+        return nil if text.nil?
+
+        text = Js.trim(text)
+        prefix = parse_prefix(text)
+        return nil if prefix.nil? || v4.is_a?(String) != (prefix[0] == 4)
+
+        text
+      end
+    end
+
+    # Downloads one published list through fetcher (anything with fetch(url, init)). Nil when it cannot be
+    # fetched or read; the caller keeps what it had.
+    def download_range_list(url, fetcher, timeout_ms = 15_000)
+      response = fetcher.fetch(url, {
+        "headers" => { "accept" => "application/json", "user-agent" => USER_AGENT },
+        "redirect" => "follow", "timeoutMs" => timeout_ms, "maxBytes" => MAX_LIST_BYTES,
+      })
+      return nil unless response.ok?
+
+      read_range_list(Body.read_json_capped(response, MAX_LIST_BYTES))
+    rescue StandardError
+      nil
+    end
+
+    def prefixes_of(list)
+      @compiled_lock.synchronize do
+        found = @compiled[list]
+        return found unless found.nil?
+
+        @compiled.clear if @compiled.length >= 256
+        @compiled[list] = list.filter_map { |p| p.is_a?(String) ? parse_prefix(p) : nil }.freeze
+      end
+    end
+
+    # Whether ip is inside any of the prefixes. False for an address that cannot be read.
+    def in_ranges?(ip, list)
+      address = parse_address(ip)
+      !address.nil? && prefixes_of(list).any? { |p| covers?(p, address) }
+    end
+
+    # The check for one request that names agent (a Hash from Data::Agents), made from ip (see the top of
+    # this file): "verified", "mismatch", or "".
+    def check_agent(agent, ip, lists)
+      list = agent["ranges"].nil? ? nil : lists[agent["ranges"]]
+      return "" if list.nil? || parse_address(ip).nil?
+
+      in_ranges?(ip, list) ? "verified" : "mismatch"
+    end
+
+    private_class_method :ipv4, :ipv6, :parse_prefix, :covers?, :prefixes_of
+  end
+end

data/lib/runlight/core.rb

@@ -42,6 +42,9 @@ module Runlight
   # - localInstalls: lets a connected install be at http://localhost or http://127.0.0.1, for trying a hub and
   #   an app on one machine. Default false: otherwise anyone who can add a site could have this server ask
   #   services on its own machine, so other installs must be public https addresses.
+  # - updateAgentRanges: downloads the address lists AI companies publish, at most once a day during check(), so
+  #   requests that name an agent are checked against current addresses. Default false: the lists that came
+  #   with this version are used, and Runlight makes no requests of its own for them.
   # - now: a callable giving the clock in milliseconds. For tests.
   # - fetcher: what every outgoing request goes through (anything with fetch(url, init)). Default
   #   Http::NetFetcher.
@@ -72,7 +75,9 @@ module Runlight
     ROLLUP_DELAY_MS = 2 * 3_600_000
     # The longest a tracker hit waits for the tables while another process creates or upgrades them.
     TRACKER_WAIT_MS = 10_000
-    private_constant :ROLLUP_VERSION, :ROLLUP_BATCH, :REMOTE_MAX_BYTES, :METERED_ROLLUP_BATCH, :ROLLUP_DELAY_MS, :TRACKER_WAIT_MS
+    # How often the scheduled check downloads the agents' address lists, when updateAgentRanges is on.
+    AGENT_RANGES_EVERY_MS = 86_400_000
+    private_constant :AGENT_RANGES_EVERY_MS, :ROLLUP_VERSION, :ROLLUP_BATCH, :REMOTE_MAX_BYTES, :METERED_ROLLUP_BATCH, :ROLLUP_DELAY_MS, :TRACKER_WAIT_MS
 
     BUSY = /timeout exceeded when trying to connect|connection timeout|no MySQL connection was free|SQLITE_BUSY|database is locked/i
     private_constant :BUSY
@@ -164,6 +169,10 @@ module Runlight
                        else Mail::Transports::SERVICES.select { |s| offered.include?(s["id"]) }.freeze
                        end
       @local_installs = options["localInstalls"] ? true : false
+      @update_agent_ranges = Js.truthy?(options["updateAgentRanges"])
+      # The agents' address lists in use: downloaded ones from the database when there are any, else those that
+      # came with this version.
+      @agent_ranges = AgentCheck.bundled["lists"]
       @secret = if options.key?("secret") && !options["secret"].nil?
                   options["secret"].to_s
                 else
@@ -961,7 +970,12 @@ module Runlight
     # Records a request from a known AI agent. Call it from middleware for
     # every page request; it ignores everything else and never raises.
     # Agents do not run JavaScript, so the tracker cannot see them.
-    def observe(request, at = nil)
+    #
+    # The request's address (from context "ip" or a trusted proxy header, as
+    # for tracker hits, or context "agentIp" when a log reader or plugin
+    # reports it) is checked against the list the agent's company publishes
+    # and then dropped: only "verified", "mismatch", or "" is kept with the fetch.
+    def observe(request, at = nil, context = {})
       return false if request.method != "GET"
 
       agent = Ua.ai_agent(request.headers.get("user-agent") || "")
@@ -993,9 +1007,11 @@ module Runlight
         "pageview" => "",
         "path" => Js.slice(url.pathname, 0, 1000),
         "hostname" => Sources.strip_www(url.hostname),
-        "title" => "",
         "name" => agent["name"],
         "props" => { "company" => agent["company"], "kind" => agent["kind"] },
+        # A fetch keeps the result of the address check in title: "verified", "mismatch", or "" when there was
+        # nothing to check against. The address itself is not kept.
+        "title" => AgentCheck.check_agent(agent, agent_ip(request, context), @agent_ranges),
         "engagedMs" => 0,
         "scroll" => nil,
         "link" => "",
@@ -1502,6 +1518,61 @@ module Runlight
       })
     end
 
+    # The address an AI agent's request is checked from: the one a report names, else the client's.
+    def agent_ip(request, context)
+      given = context.key?("agentIp") ? context["agentIp"] : context[:agent_ip]
+      given.nil? ? client_ip(request, context) : given.to_s
+    end
+    private :agent_ip
+
+    # Uses the agents' address lists downloaded by any process sharing this database, and downloads them again
+    # when updateAgentRanges is on and the last try was a day ago or more. A list that cannot be fetched or read
+    # keeps the copy it replaces; a failed day is not retried until the next day.
+    def refresh_agent_ranges
+      saved = saved_agent_ranges(@store.setting("agent-ranges"))
+      if @update_agent_ranges && (saved.nil? || now - saved["at"] >= AGENT_RANGES_EVERY_MS)
+        lists = saved.nil? ? {} : saved["lists"].dup
+        AgentCheck.bundled["lists"].each_key do |url|
+          fresh = AgentCheck.download_range_list(url, @fetcher)
+          lists[url] = fresh unless fresh.nil?
+        end
+        saved = { "at" => now, "lists" => lists }
+        @store.set_setting("agent-ranges", Json.encode(saved))
+      end
+      @agent_ranges = saved.nil? ? AgentCheck.bundled["lists"] : AgentCheck.bundled["lists"].merge(saved["lists"])
+    rescue StandardError => e
+      warn("Runlight: could not update the AI agents' address lists #{e.message}")
+    end
+    private :refresh_agent_ranges
+
+    # The "agent-ranges" setting, { "at" => ms, "lists" => { url => [prefix] } } with each list read as a
+    # download is, or nil when there is none or it is not JSON. A list that does not read is left out.
+    def saved_agent_ranges(stored)
+      return nil if stored.nil? || stored == ""
+
+      parsed = begin
+        Json.decode(stored)
+      rescue StandardError
+        return nil
+      end
+      return nil if parsed.nil?
+
+      given = parsed.is_a?(Hash) ? parsed["lists"] : nil
+      entries = if given.is_a?(Hash) then given.to_a
+                elsif given.is_a?(Array) then given.each_with_index.map { |list, i| [i.to_s, list] }
+                else []
+                end
+      lists = {}
+      entries.each do |url, list|
+        wrapped = list.is_a?(Array) ? list.map { |p| { (Js.string(p).include?(":") ? "ipv6Prefix" : "ipv4Prefix") => p } } : nil
+        prefixes = AgentCheck.read_range_list({ "prefixes" => wrapped })
+        lists[url] = prefixes unless prefixes.nil?
+      end
+      at = parsed.is_a?(Hash) ? Js.number(parsed["at"]) : 0
+      { "at" => at.is_a?(Float) && (at.nan? || at.zero?) ? 0 : at, "lists" => lists }
+    end
+    private :saved_agent_ranges
+
     def run_check
       init
       # Requests take a current schema version on trust; the scheduled check goes over every table and index.
@@ -1512,6 +1583,7 @@ module Runlight
       end
       # A name or timezone changed in the dashboard by another process reaches this one too.
       @overrides = @store.site_overrides
+      refresh_agent_ranges
       @salts = {}
       sites.map { |site| site["timezone"] }.uniq.each { |timezone| current_salts(now, timezone) }
       drop_old_salts(now)

data/lib/runlight/data/agents.rb

@@ -8,27 +8,34 @@ module Runlight
     # "live" means an assistant fetched the page because a person asked about it
     # just now. "crawl" is training or search indexing.
     #
-    # Each agent is a Hash {"name", "company", "kind" ("live" or "crawl"), "token"}; the token is matched
-    # case-insensitively as a substring of the user agent.
+    # Each agent is a Hash {"name", "company", "kind" ("live" or "crawl"), "token", "ranges"}; the token is matched
+    # case-insensitively as a substring of the user agent. "ranges" is the list of addresses its company
+    # publishes for it, which a request naming it is checked against (see AgentCheck); it is absent when the
+    # company publishes none, so its requests stay claimed.
     module Agents
+      OPENAI = "https://openai.com/"
+      ANTHROPIC = "https://claude.com/crawling/bots.json"
+      PERPLEXITY = "https://www.perplexity.ai/"
+      private_constant :OPENAI, :ANTHROPIC, :PERPLEXITY
+
       AI_AGENTS = [
-        { "name" => "ChatGPT-User", "company" => "OpenAI", "kind" => "live", "token" => "chatgpt-user" },
-        { "name" => "OAI-SearchBot", "company" => "OpenAI", "kind" => "crawl", "token" => "oai-searchbot" },
-        { "name" => "GPTBot", "company" => "OpenAI", "kind" => "crawl", "token" => "gptbot" },
-        { "name" => "Claude-User", "company" => "Anthropic", "kind" => "live", "token" => "claude-user" },
-        { "name" => "Claude-SearchBot", "company" => "Anthropic", "kind" => "crawl", "token" => "claude-searchbot" },
-        { "name" => "ClaudeBot", "company" => "Anthropic", "kind" => "crawl", "token" => "claudebot" },
-        { "name" => "Claude-Web", "company" => "Anthropic", "kind" => "crawl", "token" => "claude-web" },
-        { "name" => "anthropic-ai", "company" => "Anthropic", "kind" => "crawl", "token" => "anthropic-ai" },
-        { "name" => "Perplexity-User", "company" => "Perplexity", "kind" => "live", "token" => "perplexity-user" },
-        { "name" => "PerplexityBot", "company" => "Perplexity", "kind" => "crawl", "token" => "perplexitybot" },
-        { "name" => "MistralAI-User", "company" => "Mistral", "kind" => "live", "token" => "mistralai-user" },
+        { "name" => "ChatGPT-User", "company" => "OpenAI", "kind" => "live", "token" => "chatgpt-user", "ranges" => "#{OPENAI}chatgpt-user.json" },
+        { "name" => "OAI-SearchBot", "company" => "OpenAI", "kind" => "crawl", "token" => "oai-searchbot", "ranges" => "#{OPENAI}searchbot.json" },
+        { "name" => "GPTBot", "company" => "OpenAI", "kind" => "crawl", "token" => "gptbot", "ranges" => "#{OPENAI}gptbot.json" },
+        { "name" => "Claude-User", "company" => "Anthropic", "kind" => "live", "token" => "claude-user", "ranges" => ANTHROPIC },
+        { "name" => "Claude-SearchBot", "company" => "Anthropic", "kind" => "crawl", "token" => "claude-searchbot", "ranges" => ANTHROPIC },
+        { "name" => "ClaudeBot", "company" => "Anthropic", "kind" => "crawl", "token" => "claudebot", "ranges" => ANTHROPIC },
+        { "name" => "Claude-Web", "company" => "Anthropic", "kind" => "crawl", "token" => "claude-web", "ranges" => ANTHROPIC },
+        { "name" => "anthropic-ai", "company" => "Anthropic", "kind" => "crawl", "token" => "anthropic-ai", "ranges" => ANTHROPIC },
+        { "name" => "Perplexity-User", "company" => "Perplexity", "kind" => "live", "token" => "perplexity-user", "ranges" => "#{PERPLEXITY}perplexity-user.json" },
+        { "name" => "PerplexityBot", "company" => "Perplexity", "kind" => "crawl", "token" => "perplexitybot", "ranges" => "#{PERPLEXITY}perplexitybot.json" },
+        { "name" => "MistralAI-User", "company" => "Mistral", "kind" => "live", "token" => "mistralai-user", "ranges" => "https://mistral.ai/mistralai-user-ips.json" },
         { "name" => "meta-externalfetcher", "company" => "Meta", "kind" => "live", "token" => "meta-externalfetcher" },
         { "name" => "meta-externalagent", "company" => "Meta", "kind" => "crawl", "token" => "meta-externalagent" },
-        { "name" => "DuckAssistBot", "company" => "DuckDuckGo", "kind" => "crawl", "token" => "duckassistbot" },
+        { "name" => "DuckAssistBot", "company" => "DuckDuckGo", "kind" => "crawl", "token" => "duckassistbot", "ranges" => "https://duckduckgo.com/duckassistbot.json" },
         { "name" => "Amazonbot", "company" => "Amazon", "kind" => "crawl", "token" => "amazonbot" },
         { "name" => "Bytespider", "company" => "ByteDance", "kind" => "crawl", "token" => "bytespider" },
-        { "name" => "CCBot", "company" => "Common Crawl", "kind" => "crawl", "token" => "ccbot" },
+        { "name" => "CCBot", "company" => "Common Crawl", "kind" => "crawl", "token" => "ccbot", "ranges" => "https://index.commoncrawl.org/ccbot.json" },
         { "name" => "cohere-ai", "company" => "Cohere", "kind" => "crawl", "token" => "cohere-ai" },
         { "name" => "YouBot", "company" => "You.com", "kind" => "crawl", "token" => "youbot" },
         { "name" => "Diffbot", "company" => "Diffbot", "kind" => "crawl", "token" => "diffbot" },

data/lib/runlight/routes.rb

@@ -2090,7 +2090,7 @@ module Runlight
       body = read_json(request)
       return body if body.is_a?(Http::Response)
 
-      # One fetch as { url, userAgent, at? }, or up to 500 as { fetches: [...] } from a log reader.
+      # One fetch as { url, userAgent, at?, ip? }, or up to 500 as { fetches: [...] } from a log reader.
       fetches = Js.get(body, "fetches")
       batch = fetches.is_a?(Array)
       list = batch ? fetches : [body]
@@ -2110,10 +2110,14 @@ module Runlight
                      Importers::Client.parse_date(at)
                    end
         agent = read.call("userAgent")
+        # ip is the address the agent's request came from, checked against its company's list and then dropped.
+        ip = read.call("ip")
+        ip = ip.is_a?(String) ? Js.slice(ip, 0, 100) : ""
         pages << {
           "page" => page,
           "userAgent" => Js.slice(agent.nil? || agent.equal?(UNDEFINED) ? "" : Js.string(agent), 0, 500),
           "at" => !whenever.nil? && whenever.to_f.finite? ? whenever : nil,
+          "ip" => ip,
         }
       end
       keep = pages
@@ -2125,8 +2129,10 @@ module Runlight
         return coded("Unauthorized", "unauthorized", 401) if !batch && keep.empty?
       end
       recorded = 0
+      # A fetch reported without its address cannot be checked; it is never checked against the reporter's own.
       keep.each do |p|
-        recorded += 1 if rl.observe(Http::Request.new(p["page"].href, method: "GET", headers: { "user-agent" => p["userAgent"] }), p["at"])
+        request = Http::Request.new(p["page"].href, method: "GET", headers: { "user-agent" => p["userAgent"] })
+        recorded += 1 if rl.observe(request, p["at"], { "agentIp" => p["ip"] })
       end
       # A single report, as the CMS plugins send, needs no answer; a batch learns what was kept.
       return Http::Response.new("", status: 204) unless batch

data/lib/runlight/server/agents.rb

@@ -25,7 +25,7 @@ module Runlight
       S = "[^#{Js::SPACE}]"
 
       # host? ip - user [time] "METHOD /path HTTP/x" status bytes "referrer" "user agent"
-      COMBINED = /\A(?:(#{S}+) )?#{S}+ #{S}+ #{S}+ \[([^\]]+)\] "(#{S}+) (#{S}+)[^"]*" ([0-9]{3}) #{S}+ "(?:[^"\\]|\\[^\n\r

])*" "((?:[^"\\]|\\[^\n\r

])*)"/
+      COMBINED = /\A(?:(#{S}+) )?(#{S}+) #{S}+ #{S}+ \[([^\]]+)\] "(#{S}+) (#{S}+)[^"]*" ([0-9]{3}) #{S}+ "(?:[^"\\]|\\[^\n\r

])*" "((?:[^"\\]|\\[^\n\r

])*)"/
 
       # A request line and status, in a line that might be a combined log line without its host.
       REQUEST = %r{"#{S}+ /#{S}* [^"]*" [0-9]{3}}
@@ -137,8 +137,15 @@ module Runlight
             url = uri.is_a?(String) ? page_url(uri, host) : nil
             return nil if url.nil?
 
+            # client_ip is the visitor's address after Caddy's trusted proxies; remote_ip is the connection's.
+            client = at(request, "client_ip")
+            remote = at(request, "remote_ip")
+            ip = if client.is_a?(String) then client
+                 elsif remote.is_a?(String) then remote
+                 else ""
+                 end
             return { "method" => method, "url" => url, "status" => Js.number(either(at(entry, "status"), 0)),
-                     "userAgent" => Js.string(ua), "at" => at_ms }
+                     "userAgent" => Js.string(ua), "at" => at_ms, "ip" => ip }
           rescue Json::ParseError, TypeError
             return nil
           end
@@ -154,11 +161,11 @@ module Runlight
         base = vhost.nil? ? site : "https://#{vhost}"
         return nil if base.nil? || base == ""
 
-        url = page_url(m[4], base)
+        url = page_url(m[5], base)
         return nil if url.nil?
 
-        { "method" => m[3], "url" => url, "status" => m[5].to_i, "userAgent" => m[6].gsub('\\"', '"'),
-          "at" => log_time(m[2]) }
+        { "method" => m[4], "url" => url, "status" => m[6].to_i, "userAgent" => m[7].gsub('\\"', '"'),
+          "at" => log_time(m[3]), "ip" => m[2] }
       end
 
       # The lines worth sending: GETs that succeeded, from known AI agents. now gives epoch milliseconds, for a
@@ -170,7 +177,10 @@ module Runlight
 
         at = hit["at"]
         finite = !at.is_a?(Float) || at.finite?
-        { "url" => hit["url"], "userAgent" => hit["userAgent"], "at" => finite ? at : (now || method(:clock)).call }
+        fetch = { "url" => hit["url"], "userAgent" => hit["userAgent"], "at" => finite ? at : (now || method(:clock)).call }
+        # The address goes along so the server can check the agent's published list.
+        fetch["ip"] = hit["ip"] unless hit["ip"].nil? || hit["ip"] == ""
+        fetch
       end
 
       def clock

data/lib/runlight/server/cli.rb

@@ -24,8 +24,12 @@ module Runlight
         Add --config <file> to read settings from a config.rb other than the
         project folder's. Settings are read from the environment first and then
         config.rb: PORT, HOST, DATA_DIR, DATABASE_URL, RUNLIGHT_SECRET,
-        RUNLIGHT_TOKEN, RUNLIGHT_URL, TRUST_PROXY, RUNLIGHT_GEO, CRON_SECRET, and
-        RUNLIGHT_OBSERVE_KEY.
+        RUNLIGHT_TOKEN, RUNLIGHT_URL, TRUST_PROXY, RUNLIGHT_GEO, CRON_SECRET,
+        RUNLIGHT_OBSERVE_KEY, and RUNLIGHT_AGENT_RANGES.
+
+        RUNLIGHT_AGENT_RANGES=off stops the daily download of the address lists AI
+        companies publish, which AI agent fetches are checked against; the lists that
+        came with this version are used instead.
 
         The server runs the scheduled check every five minutes. To run it from
         cron as well, or for a server that is not always running:

data/lib/runlight/server/config.rb

@@ -21,6 +21,8 @@ module Runlight
     #   RUNLIGHT_GEO          city (the default), country, off, or the path to an MMDB file
     #   CRON_SECRET           lets a scheduler run the check over HTTP, at POST /api/check
     #   RUNLIGHT_OBSERVE_KEY  one key for every site's AI agent reports
+    #   RUNLIGHT_AGENT_RANGES on (the default) downloads the AI agents' published address lists daily; off
+    #                         keeps the lists that came with this version
     #
     # Relative paths are read from the project folder.
     class Config
@@ -117,6 +119,16 @@ module Runlight
 
       # false with nothing in front, or the one header your proxy sets, such as cf-connecting-ip behind Cloudflare.
       # Unset stays nil, so the library's default applies: forwarding headers count only from a private address.
+      # Whether the scheduled check downloads the AI agents' address lists: RUNLIGHT_AGENT_RANGES on (the
+      # default) or off.
+      def agent_ranges
+        value = get("RUNLIGHT_AGENT_RANGES")
+        return true if value.nil? || value == "on"
+        return false if value == "off"
+
+        raise RuntimeError, "Runlight: set RUNLIGHT_AGENT_RANGES to on or off"
+      end
+
       def trust_proxy
         value = (get("TRUST_PROXY") || "").downcase
         return nil if value == ""
@@ -196,6 +208,7 @@ module Runlight
           "geoCredit" => !db_ip.nil?,
           "cronSecret" => get("CRON_SECRET"),
           "observeKey" => get("RUNLIGHT_OBSERVE_KEY"),
+          "updateAgentRanges" => agent_ranges,
         }
         settings["geo"] = geo unless geo.nil?
         unless code.nil?

data/lib/runlight/server/standalone.rb

@@ -58,6 +58,8 @@ module Runlight
       #   written in setup.txt, for the page that asks for it.
       # - cronSecret: a bearer secret for POST /api/check, for a scheduler that calls it over HTTP.
       # - observeKey: one key for every site's AI agent reports.
+      # - updateAgentRanges: download the address lists AI companies publish once a day, to check AI agent fetches
+      #   against. Default false; Config turns it on unless RUNLIGHT_AGENT_RANGES is off.
       def initialize(options)
         options = Options.normalize(options)
         @store = options["store"]
@@ -80,6 +82,7 @@ module Runlight
         core["trustProxy"] = @trust_proxy unless @trust_proxy.nil?
         core["geo"] = options["geo"] unless options["geo"].nil?
         core["fetcher"] = options["fetcher"] unless options["fetcher"].nil?
+        core["updateAgentRanges"] = options["updateAgentRanges"] ? true : false
         @runlight = rl = Core.new(core)
 
         # Accounts, shared with apps that turn them on. The first one is made with the one-time code, or with the

data/lib/runlight/store/sql_store.rb

@@ -1284,12 +1284,24 @@ module Runlight
       def breakdown(query, dimension, limit, offset)
         page = [limit, offset]
         dialect = @db.dialect
-        if dimension == "ai_agent" || dimension == "ai_page"
-          col = dimension == "ai_agent" ? "e.name" : "e.path"
+        if dimension == "ai_agent"
+          # A fetch's title holds its address check (see Core#observe); only "verified" is not claimed.
+          claimed = "CASE WHEN e.title = 'verified' THEN 0 ELSE 1 END"
           rows = @db.all(
-            "SELECT #{col} AS value, COUNT(*) AS fetches FROM rl_events e
+            "SELECT e.name AS value, #{claimed} AS claimed, COUNT(*) AS fetches FROM rl_events e
          WHERE e.site = ? AND e.ts >= ? AND e.ts < ? AND e.kind = 'fetch'
-         GROUP BY #{col} ORDER BY fetches DESC, #{col}#{text_order} LIMIT ? OFFSET ?",
+         GROUP BY e.name, #{claimed} ORDER BY fetches DESC, e.name#{text_order}, claimed LIMIT ? OFFSET ?",
+            [query["site"], query["from"], query["to"], *page],
+          )
+          return rows.map do |row|
+            { "value" => Sql.string(row["value"]), "visitors" => 0, "fetches" => Sql.num(row["fetches"]), "claimed" => Sql.num(row["claimed"]) == 1 }
+          end
+        end
+        if dimension == "ai_page"
+          rows = @db.all(
+            "SELECT e.path AS value, COUNT(*) AS fetches FROM rl_events e
+         WHERE e.site = ? AND e.ts >= ? AND e.ts < ? AND e.kind = 'fetch'
+         GROUP BY e.path ORDER BY fetches DESC, e.path#{text_order} LIMIT ? OFFSET ?",
             [query["site"], query["from"], query["to"], *page],
           )
           return rows.map { |row| { "value" => Sql.string(row["value"]), "visitors" => 0, "fetches" => Sql.num(row["fetches"]) } }

data/lib/runlight/version.rb

@@ -4,7 +4,7 @@ require "json"
 
 module Runlight
   # The gem's own version, which the release keeps equal to the npm package's.
-  GEM_VERSION = "0.3.3"
+  GEM_VERSION = "0.4.0"
 
   # The SDK's version and the HTTP API's, read from assets/build.json, which
   # scripts/ruby-assets.mts writes from the TypeScript SDK, so the two always

data/lib/runlight.rb

@@ -13,6 +13,7 @@ require_relative "runlight/undefined"
 #
 # Every class loads on first use.
 module Runlight
+  autoload :AgentCheck, "runlight/agent_check"
   autoload :Assistant, "runlight/assistant"
   autoload :AssistantError, "runlight/assistant_error"
   autoload :Body, "runlight/body"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: runlight
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
 platform: ruby
 authors:
 - Jon C. Phillips
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2026-11-06 00:00:00.000000000 Z
+date: 2026-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -41,6 +41,7 @@ extra_rdoc_files: []
 files:
 - LICENSE
 - README.md
+- assets/agent-ranges.json
 - assets/assistant-action.json
 - assets/build.json
 - assets/dashboard.css
@@ -61,6 +62,7 @@ files:
 - lib/runlight/accounts/scrypt.rb
 - lib/runlight/accounts/throttle.rb
 - lib/runlight/accounts/web.rb
+- lib/runlight/agent_check.rb
 - lib/runlight/assistant.rb
 - lib/runlight/assistant_error.rb
 - lib/runlight/body.rb