portage-cli 0.7.5 → 0.8.0

data/lib/portage/cli/agent_profile_url.rb

@@ -0,0 +1,30 @@
+module Portage
+  module Cli
+    # Where `find`/`buy`/`OfferSources::ShopifyCatalog` get
+    # `meta.ucp-agent.profile` when the caller hasn't set
+    # PORTAGE_AGENT_PROFILE.
+    #
+    # Real UCP servers fetch and verify that URL before answering any call
+    # (Transports::Http), and the CLI used to read it via a bare
+    # `ENV.fetch("PORTAGE_AGENT_PROFILE", nil)` with no fallback — a fresh
+    # install that never copied `.env.example` got a bare
+    # MissingAgentProfileError on its very first `find`/`buy`. The repo
+    # already publishes its own profile document
+    # (`portage-cli/agent-profile/agent-profile.json`) over jsdelivr, and
+    # that URL is confirmed live against catalog.shopify.com (see
+    # docs/agent-profile.md), so it's a reasonable "works enough to try it"
+    # default — not a substitute for a caller hosting their own via
+    # `portage generate agent-profile`.
+    module AgentProfileUrl
+      DEFAULT = "https://cdn.jsdelivr.net/gh/tomtom87/Portage@main/portage-cli/agent-profile/agent-profile.json"
+                .freeze
+
+      # @return [String] PORTAGE_AGENT_PROFILE, or DEFAULT when it's unset
+      #   or blank.
+      def self.resolve
+        value = ENV.fetch("PORTAGE_AGENT_PROFILE", nil).to_s.strip
+        value.empty? ? DEFAULT : value
+      end
+    end
+  end
+end

data/lib/portage/cli/browser_import/categorize.rb

@@ -0,0 +1,59 @@
+require_relative "../classifier"
+require_relative "../index/builder"
+
+module Portage
+  module Cli
+    module BrowserImport
+      # "Smarter import" (docs/plans/buy-skill-and-local-browser.md Phase
+      # 3): a kept domain's categories come from what the user actually
+      # looked at there — each row's page title, bookmark folder name and
+      # URL slug (`/products/<slug>`, `/collections/<slug>`, `/c/<slug>`,
+      # `/category/<slug>`, split on `-`/`_`), run through the same
+      # Classifier every other index source uses, and weighted by visit
+      # count. A domain none of whose rows matches a category stays
+      # uncategorised: SearchBackends::Index then routes it by name only,
+      # never for a generic query.
+      module Categorize
+        TOP_CATEGORIES = Index::Builder::TOP_CATEGORIES
+
+        # A product page, for `--include-product-pages`: `/product/<x>` or
+        # `/products/<x>` (Shopify, WooCommerce and most others).
+        PRODUCT_PATH = %r{/products?/[^/?#]+}
+
+        # @param rows [Array<Hash>] Readers rows, all for one domain.
+        # @return [Hash{String => Integer}] category id => weight, top 5.
+        def self.domain(rows)
+          texts = Hash.new(0)
+          rows.each { |row| texts[text_of(row)] += row[:visits] }
+          tally = Hash.new(0)
+          texts.each do |text, visits|
+            Classifier.categories_for(text).each { |id| tally[id] += visits } unless text.empty?
+          end
+          tally.sort_by { |_id, weight| -weight }.first(TOP_CATEGORIES).to_h
+        end
+
+        # @param kept [Array<Hash>] Importer's kept entries, rows included.
+        # @return [Array<Hash>] one product entry per distinct product-page
+        #   title — key, title, url, origin, source ("history"/"bookmark"),
+        #   category.
+        def self.products(kept)
+          kept.flat_map { |entry| products_of(entry) }.uniq { |p| p[:key] }
+        end
+
+        def self.products_of(entry)
+          entry[:rows].select { |r| r[:url].match?(PRODUCT_PATH) && !r[:title].empty? }.map do |row|
+            { key: "title:#{row[:title].downcase.gsub(/[^a-z0-9]+/, '-')}", title: row[:title], url: row[:url],
+              origin: entry[:origin], source: row[:kind], category: Classifier.categories_for(text_of(row)).first }
+          end
+        end
+        private_class_method :products_of
+
+        def self.text_of(row)
+          slugs = Classifier::SLUG_PATTERNS.filter_map { |pattern| pattern.match(row[:url])&.[](1) }
+          [row[:title], row[:folder], *slugs.map { |s| s.tr("-_", "  ") }].reject(&:empty?).join(" ")
+        end
+        private_class_method :text_of
+      end
+    end
+  end
+end

data/lib/portage/cli/browser_import/confirm.rb

@@ -0,0 +1,35 @@
+module Portage
+  module Cli
+    module BrowserImport
+      # The "shows the list and asks before writing" gate (docs/plans/
+      # buy-skill-and-local-browser.md Phase 3), shaped like
+      # WebmcpMappingConfirm: an explicit `--yes` is the only way to save
+      # without a prompt; a prompt needs a real TTY and no `--json`; and
+      # anything else — piped, CI, an agent running `--json` — saves
+      # nothing and says how to confirm. A dry run never saves, `--yes` or
+      # not.
+      class Confirm
+        # @param interactive [Boolean] false under --json or with no TTY on
+        #   stdin (the caller decides).
+        def initialize(interactive:, input: $stdin, output: $stdout)
+          @interactive = interactive
+          @input = input
+          @output = output
+        end
+
+        # @return [Symbol] :save, :declined, :dry_run, :nothing, or
+        #   :needs_confirmation (no TTY/--json and no --yes).
+        def call(plan, yes:, dry_run:)
+          return :dry_run if dry_run
+          return :nothing if Array(plan[:kept]).empty?
+          return :save if yes
+          return :needs_confirmation unless @interactive
+
+          @output.print "Save these #{plan[:kept].length} store(s) to your local index? [y/N] "
+          @output.flush
+          @input.gets.to_s.strip.downcase == "y" ? :save : :declined
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/browser_import/domains.rb

@@ -0,0 +1,47 @@
+require "uri"
+
+module Portage
+  module Cli
+    module BrowserImport
+      # Reduces Readers rows to domains — the only thing about a row an
+      # import keeps past this point, besides the page title/folder/slug
+      # words Categorize reads locally. `www.` is folded into the bare
+      # domain, and anything that isn't an http(s) URL with a host
+      # (`chrome://`, `file:`, `about:`, `javascript:`) is dropped.
+      module Domains
+        # @return [Hash{String => Hash}] domain => { hosts: {host => visits},
+        #   rows: [...], visits: total }.
+        def self.group(rows)
+          groups = {}
+          rows.each do |row|
+            host = host_of(row[:url])
+            next unless host
+
+            group = (groups[host.delete_prefix("www.")] ||= { hosts: Hash.new(0), rows: [], visits: 0 })
+            group[:hosts][host] += row[:visits]
+            group[:rows] << row
+            group[:visits] += row[:visits]
+          end
+          groups
+        end
+
+        # The most-visited host variant (www. or not) as an https origin —
+        # UCP is https-only, so an http-only visit still probes https.
+        def self.origin_for(group) = "https://#{group[:hosts].max_by { |_h, visits| visits }.first}"
+
+        # An index entry's origin, reduced the same way, so it compares
+        # against a group's key.
+        def self.key_of(origin) = host_of(origin)&.delete_prefix("www.")
+
+        def self.host_of(url)
+          uri = URI.parse(url.to_s)
+          return nil unless %w[http https].include?(uri.scheme) && uri.host && !uri.host.empty?
+
+          uri.host.downcase
+        rescue URI::InvalidURIError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/browser_import/filter.rb

@@ -0,0 +1,91 @@
+require "ipaddr"
+
+require_relative "../search_backends"
+
+module Portage
+  module Cli
+    module BrowserImport
+      # "Is this host obviously not a shop?" — decided locally, before a
+      # single probe is spent (docs/plans/buy-skill-and-local-browser.md
+      # Phase 3: "skipping obvious non-shops (NON_STORE_HOSTS, webmail,
+      # banks, intranet and localhost hosts)"). A skipped host never leaves
+      # the machine at all, so these lists err on the side of skipping:
+      # a missed shop costs one `portage index add`, a probed bank costs
+      # the user's trust.
+      module Filter
+        WEBMAIL_HOSTS = %w[
+          mail.google.com outlook.live.com outlook.office.com outlook.office365.com mail.yahoo.com
+          proton.me protonmail.com icloud.com fastmail.com mail.aol.com gmx.com zoho.com
+        ].freeze
+
+        # Banks and payment/finance hosts are never a UCP storefront, and
+        # are exactly the hosts a shopper least wants anything sent about.
+        # Not exhaustive — backed up by BANK_WORD for the long tail.
+        BANK_HOSTS = %w[
+          paypal.com wise.com revolut.com monzo.com starlingbank.com chase.com wellsfargo.com
+          citi.com capitalone.com americanexpress.com discover.com schwab.com fidelity.com vanguard.com
+          hsbc.com hsbc.co.uk barclays.co.uk natwest.com santander.co.uk nationwide.co.uk halifax.co.uk
+          coinbase.com kraken.com stripe.com klarna.com
+        ].freeze
+
+        BANK_WORD = /bank|creditunion/
+
+        # Tools, social and media sites a shopper visits all day that will
+        # never answer /.well-known/ucp — skipping them saves probes for
+        # the domains that might.
+        NON_SHOP_HOSTS = %w[
+          github.com gitlab.com bitbucket.org stackoverflow.com stackexchange.com linkedin.com slack.com
+          zoom.us notion.so atlassian.net figma.com claude.ai anthropic.com openai.com chatgpt.com
+          instagram.com tiktok.com netflix.com spotify.com twitch.tv discord.com whatsapp.com
+          medium.com substack.com apple.com microsoft.com live.com office.com gstatic.com
+          googleusercontent.com cloudflare.com amazonaws.com
+        ].freeze
+
+        LOCAL_SUFFIXES = %w[localhost local internal lan corp home.arpa test intranet].freeze
+
+        # @return [Symbol, nil] why `host` is skipped (:non_store, :webmail,
+        #   :bank, :local), or nil when it's worth a probe.
+        def self.skip_reason(host)
+          host = host.to_s.downcase
+          return :local if local?(host)
+          return :webmail if webmail?(host)
+          return :bank if bank?(host)
+          return :non_store if non_store?(host)
+
+          nil
+        end
+
+        def self.local?(host)
+          return true if host.empty? || !host.include?(".")
+          return true if ip?(host)
+
+          LOCAL_SUFFIXES.any? { |suffix| host == suffix || host.end_with?(".#{suffix}") }
+        end
+
+        def self.webmail?(host)
+          host.start_with?("mail.", "webmail.") || matches?(host, WEBMAIL_HOSTS)
+        end
+
+        def self.bank?(host)
+          matches?(host, BANK_HOSTS) || host.match?(BANK_WORD)
+        end
+
+        # SearchBackends::NON_STORE_HOSTS (search engines, Wikipedia,
+        # social) plus this module's own NON_SHOP_HOSTS.
+        def self.non_store?(host)
+          !SearchBackends.store_candidate?("https://#{host}/") || matches?(host, NON_SHOP_HOSTS)
+        end
+
+        def self.matches?(host, list) = list.any? { |bad| host == bad || host.end_with?(".#{bad}") }
+
+        def self.ip?(host)
+          IPAddr.new(host.delete_prefix("[").delete_suffix("]"))
+          true
+        rescue IPAddr::InvalidAddressError, IPAddr::AddressFamilyError
+          false
+        end
+        private_class_method :local?, :webmail?, :bank?, :non_store?, :matches?, :ip?
+      end
+    end
+  end
+end

data/lib/portage/cli/browser_import/importer.rb

@@ -0,0 +1,248 @@
+require "uri"
+
+require_relative "../classifier"
+require_relative "../index"
+require_relative "../index/builder"
+require_relative "profiles"
+require_relative "readers"
+require_relative "filter"
+require_relative "prober"
+require_relative "categorize"
+require_relative "saver"
+require_relative "domains"
+
+module Portage
+  module Cli
+    module BrowserImport
+      # `portage browser import` (docs/plans/buy-skill-and-local-browser.md
+      # Phase 3, Tier A): bookmarks and history in, a short list of shop
+      # domains out, written to the user's own Index::Store only after the
+      # user has seen the list and said yes (Cli.run_browser_import owns
+      # that gate; this class only ever writes from #save).
+      #
+      # #plan reads the profile's allowed files (Profiles::ALLOWED_FILES,
+      # nothing else), reduces every row to its domain, and decides each
+      # domain locally first — skipped as an obvious non-shop (Filter),
+      # excluded by the user, on the hand-off-only list, or already in the
+      # index — before spending one of at most `max_probes` Prober probes
+      # on an unknown one. A domain is kept only when it answers
+      # `/.well-known/ucp`, matches a WebMCP preset, or is hand-off only.
+      # Each kept domain is classified from what the user actually looked
+      # at there (page titles, bookmark folder names, URL slugs), weighted
+      # by visit count.
+      #
+      # Kept domains land in Index::Store with `sources: ["history"]`/
+      # `["bookmark"]` — the same untrusted index every other source
+      # writes, so an imported store reaches `find` only as a
+      # `source: "index"` candidate (never Policy#merchant_allowlist, never
+      # past the `--store`/interactive pick a `--yes` buy requires), and
+      # Index::Exporter treats both labels as personal: never exported.
+      class Importer
+        DEFAULT_HISTORY_DAYS = 90
+        MAX_PROBES = 200
+
+        FULL_DISK_ACCESS = "macOS won't let this terminal read Safari's history and bookmarks without Full Disk " \
+                           "Access. To allow it, open System Settings > Privacy & Security > Full Disk Access, " \
+                           "turn it on for the terminal app you run portage from, then quit and reopen that app " \
+                           "and re-run this command. Portage doesn't try any other way in. Nothing was imported, " \
+                           "probed or saved.".freeze
+
+        # Any other browser: the same "explain and stop", without claiming
+        # to know which protection said no (macOS privacy settings, a
+        # sandboxed terminal, file permissions).
+        PERMISSION_DENIED = "Permission denied reading %<browser>s's profile folder. If your terminal runs " \
+                            "sandboxed or macOS is protecting that folder, allow access (System Settings > " \
+                            "Privacy & Security > Full Disk Access) and re-run. Portage doesn't try any other way " \
+                            "in. Nothing was imported, probed or saved.".freeze
+
+        Options = Struct.new(:browser, :root, :history_days, :include_product_pages, :max_probes, :exclude,
+                             keyword_init: true) do
+          def initialize(history_days: DEFAULT_HISTORY_DAYS, include_product_pages: false, max_probes: MAX_PROBES,
+                         exclude: [], **)
+            super
+          end
+        end
+
+        # @param handoff_only_hosts [Array<String>] Tier C hosts (Phase 5
+        #   will supply the user's own `handoff_only_hosts:` config; nothing
+        #   ships that list yet, so it defaults to empty). A history domain
+        #   on it is kept — hand-off only, never probed, never automated.
+        # @param webmcp_preset_for [#call, nil] `->(origin) { preset_or_nil }`.
+        #   Matching a WebMCP preset needs a browser bridge an import doesn't
+        #   have (same posture as Index::Sources::WebmcpSweep), so nil (the
+        #   default) skips the check entirely.
+        # @param readers [#call] `->(family) { reader }` — see Readers.for.
+        def initialize(stores: Index::Store.new, products: Index::ProductStore.new,
+                       known_cache: Index::KnownCache.new, prober: Prober.new, handoff_only_hosts: [],
+                       webmcp_preset_for: nil, readers: Readers.method(:for), now: Time.now)
+          @stores = stores
+          @products = products
+          @known_cache = known_cache
+          @prober = prober
+          @handoff_only_hosts = handoff_only_hosts.map { |h| h.to_s.downcase.delete_prefix("www.") }
+          @webmcp_preset_for = webmcp_preset_for
+          @readers = readers
+          @now = now
+        end
+
+        # @return [Hash] the proposal: counts, `kept` (domains to save) and
+        #   `products` (only with include_product_pages) — or `error:` /
+        #   `message:` when the browser's files couldn't be read (Safari
+        #   without Full Disk Access, no sqlite3, no profile found).
+        def plan(options)
+          profiles = Profiles.locate(options.browser, root: options.root)
+          return no_profile(options) if profiles.empty?
+
+          rows, opened = read_rows(profiles, since: @now.to_i - (options.history_days.to_i * 86_400))
+          summarize(options, profiles, opened, rows)
+        rescue Sqlite::PermissionDenied
+          permission_denied(options)
+        rescue Sqlite::Unavailable => e
+          { browser: options.browser, error: "reader_unavailable",
+            message: "Couldn't read the browser's files: #{e.message}." }
+        end
+
+        # Writes a #plan's kept domains (and products) into the index — see
+        # Saver.
+        # @return [Hash] stores:, products: — how many entries were written.
+        def save(proposal) = Saver.new(stores: @stores, products: @products, now: @now).save(proposal)
+
+        private
+
+        # --- Reading ---
+
+        def read_rows(profiles, since:)
+          opened = []
+          rows = profiles.flat_map do |profile|
+            reader = @readers.call(profile.family)
+            opened.concat([profile.history, profile.bookmarks].compact)
+            reader.history(profile, since: since) + reader.bookmarks(profile)
+          end
+          [rows, opened.uniq]
+        end
+
+        # --- Deciding each domain ---
+
+        def summarize(options, profiles, opened, rows)
+          groups = Domains.group(rows)
+          state = { skipped: Hash.new(0), kept: [], cached_miss: 0, not_ucp: 0, unprobed: 0 }
+          groups.sort_by { |_key, g| -g[:visits] }.each { |key, group| decide(key, group, options, state) }
+          report(options, profiles, opened, rows, groups, state)
+        end
+
+        def decide(key, group, options, state)
+          reason = local_skip_reason(key, options)
+          return state[:skipped][reason] += 1 if reason
+
+          verdict = local_verdict(key) || probe_verdict(Domains.origin_for(group), options, state)
+          state[:kept] << kept_entry(key, group, verdict) if verdict
+        end
+
+        def local_skip_reason(key, options)
+          return :excluded if Array(options.exclude).any? { |h| h.to_s.downcase.delete_prefix("www.") == key }
+
+          Filter.skip_reason(key)
+        end
+
+        # Everything decidable with no network call at all.
+        def local_verdict(key)
+          return { verdict: "handoff_only", handoff_only: true } if handoff_only?(key)
+
+          own = own_entry(key)
+          return { verdict: "indexed", origin: own["origin"] } if own
+
+          known = known_entry(key)
+          return unless known
+
+          { verdict: "known", origin: known["origin"], capabilities: known["capabilities"],
+            last_verified: known["last_verified"] }
+        end
+
+        # A cached "no UCP" verdict or the probe cap skips the probe, but
+        # not the WebMCP check (when a bridge is injected) — a page can
+        # expose WebMCP tools without any manifest at all.
+        def probe_verdict(origin, options, state)
+          session = ucp_session(origin, options, state)
+          return nil if session == :capped
+          return { verdict: "ucp", origin: origin, capabilities: Index::Builder.capabilities_of(session) } if session
+
+          preset = @webmcp_preset_for&.call(origin)
+          return { verdict: "webmcp", origin: origin, webmcp_preset: preset } if preset
+
+          state[:not_ucp] += 1 unless session == false
+          nil
+        end
+
+        # @return [Session, nil, false, :capped] a session; nil when the
+        #   probe found no UCP; false for a cached miss (no request made);
+        #   :capped once max_probes is spent.
+        def ucp_session(origin, options, state)
+          if @prober.cached_miss?(origin)
+            state[:cached_miss] += 1
+            false
+          elsif @prober.probes >= options.max_probes.to_i
+            state[:unprobed] += 1
+            :capped
+          else
+            @prober.probe(origin)
+          end
+        end
+
+        def handoff_only?(key)
+          @handoff_only_hosts.any? { |h| key == h || key.end_with?(".#{h}") }
+        end
+
+        def own_entry(key) = @stores.all.find { |e| Domains.key_of(e["origin"]) == key }
+
+        def known_entry(key) = @known_cache.stores.values.find { |e| Domains.key_of(e["origin"]) == key }
+
+        # --- The proposal ---
+
+        def kept_entry(key, group, verdict)
+          { domain: key, origin: verdict[:origin] || Domains.origin_for(group), verdict: verdict[:verdict],
+            sources: group[:rows].map { |r| r[:kind] }.uniq.sort, visits: group[:visits],
+            categories: Categorize.domain(group[:rows]), rows: group[:rows] }
+            .merge(verdict.slice(:capabilities, :webmcp_preset, :handoff_only, :last_verified))
+        end
+
+        def report(options, profiles, opened, rows, groups, state)
+          kept = state[:kept]
+          { browser: options.browser, profiles: profiles.length, files_opened: opened, rows: row_counts(rows),
+            domains: groups.length, skipped: state[:skipped].transform_keys(&:to_s),
+            already_indexed: count(kept, "indexed"), known: count(kept, "known"), probed: @prober.probes,
+            cached_miss: state[:cached_miss], not_ucp: state[:not_ucp], unprobed: state[:unprobed],
+            capped: state[:unprobed].positive?, kept: kept.map { |entry| public_entry(entry) },
+            products: options.include_product_pages ? Categorize.products(kept) : [] }
+        end
+
+        def row_counts(rows)
+          kinds = rows.map { |r| r[:kind] }
+          { history: kinds.count("history"), bookmark: kinds.count("bookmark") }
+        end
+
+        def public_entry(entry)
+          entry.except(:rows).merge(category_names: Classifier.names_for(entry[:categories].keys))
+        end
+
+        def count(kept, verdict) = kept.count { |e| e[:verdict] == verdict }
+
+        # --- Helpers ---
+
+        def no_profile(options)
+          where = options.root ? " under #{options.root}" : ""
+          { browser: options.browser, error: "no_profile",
+            message: "No #{options.browser} profile with history or bookmarks found#{where}." }
+        end
+
+        def permission_denied(options)
+          if options.browser == "safari"
+            { browser: "safari", error: "full_disk_access_required", message: FULL_DISK_ACCESS }
+          else
+            { browser: options.browser, error: "permission_denied",
+              message: format(PERMISSION_DENIED, browser: options.browser) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/browser_import/plist_xml.rb

@@ -0,0 +1,72 @@
+require "strscan"
+
+module Portage
+  module Cli
+    module BrowserImport
+      # Just enough of Apple's XML property-list format to read Safari's
+      # Bookmarks.plist once `plutil -convert xml1` has turned its binary
+      # form into XML. `plutil -convert json` would be simpler but refuses
+      # any plist holding a <date> (Safari's Reading List entries carry
+      # them), and a real XML library (rexml) isn't a runtime dependency of
+      # this gem — this is ~50 lines instead of a new one.
+      #
+      # Dates, data and numbers come back as their raw text; bookmark
+      # import only ever reads strings out of the tree.
+      module PlistXml
+        class ParseError < StandardError; end
+
+        SCALARS = "string|key|data|date|integer|real".freeze
+        ENTITIES = { "amp" => "&", "lt" => "<", "gt" => ">", "quot" => '"', "apos" => "'" }.freeze
+
+        def self.parse(xml)
+          scanner = StringScanner.new(xml.to_s)
+          raise ParseError, "not an XML plist" unless scanner.skip_until(/<plist[^>]*>/)
+
+          value(scanner)
+        end
+
+        def self.value(scanner)
+          scanner.skip(/\s*/)
+          if scanner.scan(%r{<(dict|array|#{SCALARS})\s*/>}) then empty(scanner[1])
+          elsif scanner.scan(%r{<(true|false)\s*/>}) then scanner[1] == "true"
+          elsif scanner.scan("<dict>") then dict(scanner)
+          elsif scanner.scan("<array>") then array(scanner)
+          elsif scanner.scan(%r{<(#{SCALARS})>(.*?)</\1>}m) then unescape(scanner[2])
+          else raise ParseError, "unexpected plist content at #{scanner.pos}"
+          end
+        end
+
+        def self.dict(scanner)
+          result = {}
+          until scanner.skip(%r{\s*</dict>})
+            key = value(scanner)
+            raise ParseError, "dict key isn't a string" unless key.is_a?(String)
+
+            result[key] = value(scanner)
+          end
+          result
+        end
+
+        def self.array(scanner)
+          result = []
+          result << value(scanner) until scanner.skip(%r{\s*</array>})
+          result
+        end
+
+        def self.empty(tag)
+          { "dict" => {}, "array" => [] }.fetch(tag, "")
+        end
+
+        def self.unescape(text)
+          text.gsub(/&(#x?)?(\w+);/) do
+            if Regexp.last_match(1) == "#x" then [Regexp.last_match(2).to_i(16)].pack("U")
+            elsif Regexp.last_match(1) == "#" then [Regexp.last_match(2).to_i].pack("U")
+            else ENTITIES.fetch(Regexp.last_match(2), Regexp.last_match(0))
+            end
+          end
+        end
+        private_class_method :value, :dict, :array, :empty, :unescape
+      end
+    end
+  end
+end

data/lib/portage/cli/browser_import/prober.rb

@@ -0,0 +1,60 @@
+require "timeout"
+require "portage/ucp/client"
+
+require_relative "../user_agent"
+require_relative "../probe_cache"
+
+module Portage
+  module Cli
+    module BrowserImport
+      # The only way a browser-imported domain ever leaves this machine:
+      # one `GET /.well-known/ucp` through Portage::Ucp::Client.discover
+      # (the same discovery path `find` and `index build` use), recorded
+      # in the shared ProbeCache so a domain that didn't answer last time
+      # isn't asked again. No page, title, path or visit count is ever
+      # sent — just the origin, as the host of that one request.
+      class Prober
+        THROTTLE = 0.1
+
+        # Net::HTTP's own defaults are 60s each way — fine for one store,
+        # not for 200 history domains, some of which will be dead hosts.
+        TIMEOUT = 5
+
+        # @param discover [#call, nil] `->(origin) { session_or_nil }` —
+        #   injectable so specs never make a real request; nil (the
+        #   default) uses Portage::Ucp::Client.discover.
+        def initialize(cache: ProbeCache.new, discover: nil, throttle: THROTTLE, timeout: TIMEOUT)
+          @cache = cache
+          @discover = discover || method(:ucp_discover)
+          @throttle = throttle
+          @timeout = timeout
+          @probes = 0
+        end
+
+        attr_reader :probes
+
+        # A cached "no UCP here" verdict — answered with no request at all.
+        def cached_miss?(origin) = @cache.fetch(origin) == false
+
+        # @return [Portage::Ucp::Client::Session, nil]
+        def probe(origin)
+          sleep(@throttle) if @probes.positive? && @throttle.to_f.positive?
+          @probes += 1
+          session = safely { @discover.call(origin) }
+          @cache.record(origin, !session.nil?)
+          session
+        end
+
+        private
+
+        def safely(&)
+          Timeout.timeout(@timeout, &)
+        rescue StandardError # Timeout::Error included
+          nil
+        end
+
+        def ucp_discover(origin) = Portage::Ucp::Client.discover(origin, headers: UserAgent.headers)
+      end
+    end
+  end
+end