portage-cli 0.10.0 → 0.12.0

data/lib/portage/cli/classifier/table.rb

@@ -0,0 +1,63 @@
+require "yaml"
+
+module Portage
+  module Cli
+    module Classifier
+      # The shipped categories.yml and the user's ~/.portage/categories.yml,
+      # merged (a user node replaces a shipped node of the same id), with the
+      # keyword => node ids indexes the scoring looks words up in.
+      Table = Struct.new(:nodes, :own, :parent)
+
+      class Table
+        @cache = {}
+
+        class << self
+          # Built once per pair of files and kept for the process, keyed by
+          # the files' mtime and size, so an edit to ~/.portage/categories.yml
+          # still shows up on the next call; classifying a whole catalogue
+          # would otherwise rebuild the index for every product.
+          def for(known_path, user_path)
+            key = [signature(known_path), signature(user_path)]
+            @cache.clear if @cache.size > 8
+            @cache[key] ||= build(known_path, user_path)
+          end
+
+          # @return [Hash] the YAML mapping at `path`; empty when the file is
+          #   missing, unreadable or not a mapping.
+          def load_yaml(path)
+            return {} unless path && File.readable?(path)
+
+            data = YAML.safe_load_file(path)
+            data.is_a?(Hash) ? data : {}
+          rescue StandardError
+            {}
+          end
+
+          private
+
+          def build(known_path, user_path)
+            ordered = {}
+            load_yaml(known_path).each_with_index { |(id, node), i| ordered[id] = node.merge("order" => i) }
+            load_yaml(user_path).each_with_index do |(id, node), i|
+              ordered[id] = node.merge("order" => ordered.size + i)
+            end
+            new(ordered, index(ordered, "keywords"), index(ordered, "parent_keywords"))
+          end
+
+          def index(nodes, field)
+            result = Hash.new { |hash, key| hash[key] = [] }
+            nodes.each { |id, node| Array(node[field]).each { |keyword| result[keyword] << id } }
+            result.default_proc = nil
+            result
+          end
+
+          def signature(path)
+            return nil unless path && File.readable?(path)
+
+            [path, File.mtime(path).to_r, File.size(path)]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/classifier.rb

@@ -1,4 +1,6 @@
 require "yaml"
+require_relative "classifier/table"
+require_relative "classifier/ranking"
 
 module Portage
   module Cli
@@ -18,10 +20,21 @@ module Portage
     # check matches in both directions ("carpet" contains "pet", "chair"
     # contains "hair", "scarf" contains "car") and was a real source of
     # false positives before this became whole-word.
+    #
+    # `known-stores/categories.yml` is generated by `script/categories`: a level-2
+    # node's `keywords` are its own name plus its descendants', `parent_keywords`
+    # are its parent's words, and `category-stoplist.yml` words are left out.
+    # See Ranking for how a text is scored against it.
     module Classifier
       # `known-stores/categories.yml`, from `lib/portage/cli/classifier.rb`.
       KNOWN_PATH = File.expand_path("../../../known-stores/categories.yml", __dir__).freeze
 
+      # Words that are never evidence of a category (`known-stores/category-stoplist.yml`,
+      # `word: why`): merchandising and url words, and words that name a whole family of
+      # nodes. `script/categories` leaves them out of the keywords; `categories_for`
+      # drops them from its input, so the two sides agree.
+      STOPLIST_PATH = File.expand_path("../../../known-stores/category-stoplist.yml", __dir__).freeze
+
       # The user's own additions/overrides — same id overrides a shipped
       # node's keywords, a new id extends the taxonomy. Absent by default;
       # nothing here is required for the shipped file to work.
@@ -44,21 +57,22 @@ module Portage
       # @param known_path [String] override for KNOWN_PATH — specs redirect
       #   this the same way SearchBackends::Allowlist takes its own `path:`.
       # @param user_path [String] override for PATH.
-      # @return [Array<String>] category ids, most keyword hits first. Ties
-      #   keep the shipped file's own order. Empty when nothing matches.
-      def self.categories_for(text, known_path: KNOWN_PATH, user_path: PATH)
-        words = tokenize(text)
-        return [] if words.empty?
-
-        scored = nodes(known_path, user_path).filter_map { |id, node| rank(id, node, words) }
-        scored.sort_by { |(_id, score, order)| [-score, order] }.map(&:first)
+      # @param stoplist_path [String] override for STOPLIST_PATH.
+      # @return [Array<String>] up to MAX_CATEGORIES category ids, best score
+      #   first (ties: see .tie_breakers). Empty when nothing matches.
+      def self.categories_for(text, known_path: KNOWN_PATH, user_path: PATH, stoplist_path: STOPLIST_PATH)
+        stopped = Table.load_yaml(stoplist_path)
+        counts = Ranking.word_counts(tokenize(text).reject { |word| stopped.key?(word) })
+        return [] if counts.empty?
+
+        Ranking.best(counts, Table.for(known_path, user_path), stopped)
       end
 
       # @return [Array<String>] the taxonomy names for `ids`, in order —
       #   `portage browser import` (Phase 3) shows a domain's guessed
       #   categories by name so the user can judge them before saving.
       def self.names_for(ids, known_path: KNOWN_PATH, user_path: PATH)
-        all = nodes(known_path, user_path)
+        all = Table.for(known_path, user_path).nodes
         Array(ids).filter_map { |id| all.dig(id.to_s, "name") }
       end
 
@@ -88,17 +102,6 @@ module Portage
                                                     .select { |word| word.length >= MIN_WORD_LENGTH }
       end
 
-      # --- Scoring one node against the tokenized input ---
-
-      def self.rank(id, node, words)
-        keywords = Array(node["keywords"])
-        hits = keywords.count { |keyword| words.any? { |word| word_match?(word, keyword) } }
-        return nil unless hits.positive?
-
-        [id, hits, node["order"].to_i]
-      end
-      private_class_method :rank
-
       # Whole-word only, plus the plural forms a keyword list and a real
       # query/title actually differ by: an exact match, one plus a trailing
       # "s" or "es" ("boot"/"boots", "watch"/"watches"), or the "y"/"ies"
@@ -129,30 +132,6 @@ module Portage
           (keyword.end_with?("ies") && word == "#{keyword[0..-4]}y")
       end
       private_class_method :ies_y_match?
-
-      # --- Loading and merging the two files ---
-
-      # Re-read on every call rather than cached process-wide: `find` calls
-      # this once or twice per invocation, not in a hot loop, and a cached
-      # copy would miss an edit to ~/.portage/categories.yml until the next
-      # process start.
-      def self.nodes(known_path, user_path)
-        ordered = {}
-        load_yaml(known_path).each_with_index { |(id, node), i| ordered[id] = node.merge("order" => i) }
-        load_yaml(user_path).each_with_index { |(id, node), i| ordered[id] = node.merge("order" => ordered.size + i) }
-        ordered
-      end
-      private_class_method :nodes
-
-      def self.load_yaml(path)
-        return {} unless path && File.readable?(path)
-
-        data = YAML.safe_load_file(path)
-        data.is_a?(Hash) ? data : {}
-      rescue StandardError
-        {}
-      end
-      private_class_method :load_yaml
     end
   end
 end

data/lib/portage/cli/confidence_check.rb

@@ -8,7 +8,9 @@ module Portage
     # § Responsibilities 3) as `portage buy` uses it: one yes/no question put
     # to a Decision::ModelBackends backend right before an unattended
     # (`--yes`) completion — "is this checkout what the shopper asked for,
-    # and safe to complete without a person looking at it?"
+    # and safe to complete without a person looking at it?" Also asked
+    # before the WebMCP preset flow sends the browser to the store's
+    # checkout and autofills it (Buy#webmcp_cart_held_report).
     #
     # Default off. Nothing asks a model anything unless a backend is named,
     # via `--decision-backend NAME` or PORTAGE_DECISION_BACKEND (`jev` or
@@ -22,6 +24,16 @@ module Portage
     # purchase and hands the checkout to the shopper. It never
     # auto-approves anything `--yes` wouldn't already have allowed.
     #
+    # Additive only. Buy runs it after #reconcile_checkout's deterministic
+    # mismatch check, the quote cap and the spend policy, and only on a
+    # checkout all three let through, so a "yes" here can never let
+    # through a purchase one of those would stop. A "no" can only hold it.
+    #
+    # What it sends is ConfidenceState's allowlisted summary, never the
+    # raw checkout: no payment token, no address, no buyer contact
+    # details. #call JSON-encodes whatever state it is given, so a caller
+    # outside Buy has to build its state the same way.
+    #
     # Fails closed. An unknown backend name, a backend that isn't configured
     # (no JEV_API_KEY, no Laya bridge), or a backend call that fails all hold
     # the purchase, the same as a low score does. So does naming a backend
@@ -35,11 +47,18 @@ module Portage
       QUESTION = "safe_to_complete".freeze
       # Phrased so "yes" means "proceed": a noul answer's value is the
       # probability of yes, and that is what ConfidenceGate thresholds.
-      INSTRUCTIONS = "The state is a checkout an agent built on a shopper's behalf: the shopper's search " \
-                     "query, the merchant, the requested quantity, the checkout's line items and totals, and " \
-                     "any warnings about where the checkout differs from the request. Answer yes only if the " \
-                     "checkout clearly matches what the shopper asked for and is safe to complete without a " \
-                     "person reviewing it first.".freeze
+      INSTRUCTIONS = "The state is a summary of a checkout an agent built on a shopper's behalf. `request` " \
+                     "is what the shopper asked for: the search query, the store, the quantity, and the item " \
+                     "picked from the store's search results. `approved_quote`, when present, is the exact " \
+                     "purchase the shopper approved beforehand: store, product, title, quantity, total and " \
+                     "currency. `checkout` is what the store is about to charge for: its line items (`requested` " \
+                     "marks the line that was asked for), totals including any shipping, tax and fees, " \
+                     "discounts and the selected shipping option. `warnings` lists differences an automatic " \
+                     "check already found. Answer yes only if the checkout clearly matches the request and, " \
+                     "when there is one, the approved quote: the same product (not a different model, size, " \
+                     "bundle or subscription), the same quantity, no extra items, and a total with no " \
+                     "surprising shipping, fees or other charges. Answer no if anything is unclear, missing " \
+                     "or doesn't fit, or if the checkout would need a person to review it before paying.".freeze
 
       # @param backend [String, nil] a ModelBackends::REGISTRY key; nil
       #   defers to PORTAGE_DECISION_BACKEND.
@@ -64,7 +83,8 @@ module Portage
 
       def enabled? = !@backend_name.nil?
 
-      # @param state [Hash] JSON-serializable. Never pass it a payment token.
+      # @param state [Hash] JSON-serializable — ConfidenceState.build's
+      #   output. Never a payment token, address or contact detail.
       # @return [Hash, nil] nil when disabled. Otherwise `proceed:`,
       #   `reason:`, `confidence:`, `threshold:`, `backend:` and `error:`.
       #   `reason` is nil when it proceeds, else why it held:

data/lib/portage/cli/confidence_state.rb

@@ -0,0 +1,109 @@
+require "portage/ucp"
+
+module Portage
+  module Cli
+    # The state ConfidenceCheck sends to its model backend, which may be a
+    # hosted third-party API (Jev, run by TypeSafe). Built from an allowlist:
+    # every field below is copied by name, and nothing else in the checkout
+    # hash is ever read. A store can put a buyer block, a shipping address,
+    # payment handlers or anything else on its checkout, and none of it
+    # reaches the backend, because this never copies a sub-hash wholesale.
+    #
+    # Never sent: the payment token (or any token reference), the shipping
+    # address and destinations, the buyer's name, phone or email, links and
+    # continue URLs, ids of the checkout itself, image URLs, discount codes,
+    # and anything read from the environment.
+    #
+    # Sent:
+    # - `request`: the search query, the store's host, the requested
+    #   quantity, and the item id and title that were picked from the
+    #   store's own search results.
+    # - `approved_quote` (only on `buy --quote`): the store, product id,
+    #   title, quantity, total and currency the person approved.
+    # - `checkout`: its status and currency; each line's item id, title,
+    #   unit price, quantity, line totals and whether it is the requested
+    #   line; the checkout's totals (subtotal, shipping, tax, fees, total:
+    #   whatever types the store sends); applied discounts' titles and
+    #   amounts; and the title and price of each selected shipping option.
+    # - `warnings`: the deterministic check's warnings, if any.
+    #
+    # Strings are cut to MAX_STRING characters and anything that isn't a
+    # string, number, boolean or nil is dropped, so a store can't smuggle a
+    # nested object (or a very long prompt) in through a title.
+    module ConfidenceState
+      MAX_STRING = 200
+
+      module_function
+
+      # @param request [Hash] `query:`, `merchant:`, `quantity:`,
+      #   `item_id:`, `item_title:`.
+      # @param checkout [Hash] a checkout or cart wire hash, string-keyed.
+      # @param warnings [Array<String>]
+      # @param quote [Hash, nil] `store:`, `product_id:`, `title:`,
+      #   `quantity:`, `total:`, `currency:` — nil when the run has no
+      #   approved quote.
+      # @return [Hash] JSON-serializable, string-keyed.
+      def build(request:, checkout:, warnings:, quote: nil)
+        state = { "request" => pick(request, %i[query merchant quantity item_id item_title]) }
+        state["approved_quote"] = pick(quote, %i[store product_id title quantity total currency]) if quote
+        state["checkout"] = checkout_summary(checkout, request[:item_id])
+        state["warnings"] = Array(warnings).map { |warning| scalar(warning.to_s) }
+        state
+      end
+
+      def checkout_summary(checkout, requested_id)
+        { "status" => scalar(checkout["status"]), "currency" => scalar(checkout["currency"]),
+          "line_items" => Array(checkout["line_items"]).map { |line| line_summary(line, requested_id) },
+          "totals" => totals_summary(checkout["totals"]),
+          "discounts" => discounts_summary(checkout["discounts"]),
+          "shipping" => shipping_summary(checkout["fulfillment"]) }
+      end
+
+      def line_summary(line, requested_id)
+        line = {} unless line.is_a?(Hash)
+        item = line["item"].is_a?(Hash) ? line["item"] : {}
+        { "item_id" => scalar(item["id"]), "title" => scalar(item["title"]), "unit_price" => scalar(item["price"]),
+          "quantity" => scalar(line["quantity"]), "totals" => totals_summary(line["totals"]),
+          "requested" => !requested_id.nil? && item["id"] == requested_id }
+      end
+
+      def totals_summary(totals)
+        hashes(totals).map { |total| { "type" => scalar(total["type"]), "amount" => scalar(total["amount"]) } }
+      end
+
+      def discounts_summary(discounts)
+        applied = discounts.is_a?(Hash) ? discounts["applied"] : nil
+        hashes(applied).map do |discount|
+          { "title" => scalar(discount["title"]), "amount" => scalar(discount["amount"]) }
+        end
+      end
+
+      # Only the options the checkout has selected — the price the person
+      # pays for shipping — never the destinations they're priced for.
+      def shipping_summary(fulfillment)
+        methods = fulfillment.is_a?(Hash) ? fulfillment["methods"] : nil
+        hashes(methods).flat_map { |method| hashes(method["groups"]) }.filter_map do |group|
+          option = hashes(group["options"]).find { |o| o["id"] == group["selected_option_id"] }
+          next unless option
+
+          { "title" => scalar(option["title"]),
+            "amount" => scalar(Portage::Ucp::Support::Totals.amount(hashes(option["totals"]))) }
+        end
+      end
+
+      def pick(hash, keys)
+        keys.to_h { |key| [key.to_s, scalar(hash[key])] }
+      end
+
+      def hashes(value) = Array(value).grep(Hash)
+
+      def scalar(value)
+        case value
+        when String then value[0, MAX_STRING]
+        when Integer, true, false, nil then value
+        when Float then value.finite? ? value : nil
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/doctor.rb

@@ -42,7 +42,8 @@ module Portage
       #   those four warnings were noise on every fresh install.
       def initialize(adapter_class: nil, proxy_settings: ProxySettings.new, install_doctor: InstallDoctor.new,
                      seller: true, dot_env_path: DotEnv.loaded_path, index_stores: Index::Store.new,
-                     index_products: Index::ProductStore.new, known_cache: Index::KnownCache.new)
+                     index_products: Index::ProductStore.new, known_cache: Index::KnownCache.new,
+                     index_database: Index::Database.new(path: Index::Database.path_for(Index::Store::PATH)))
         @adapter_class = adapter_class
         @proxy_settings = proxy_settings
         @install_doctor = install_doctor
@@ -51,6 +52,7 @@ module Portage
         @index_stores = index_stores
         @index_products = index_products
         @known_cache = known_cache
+        @index_database = index_database
       end
 
       # `portage setup` always offers its wizard on a TTY; a bare `portage
@@ -210,12 +212,20 @@ module Portage
       # leaves behind.
       def index_finding
         refresh_known_cache_if_stale
-        return Finding.new(check: "index", level: "info", message: "#{index_message}\n#{known_cache_message}") \
-          if @index_stores.exists?
+        first = @index_stores.exists? ? index_message : no_index_message
+        database = @index_database.info
+        Finding.new(check: "index", level: "info", details: { database: database },
+                    message: [first, database_message(database), known_cache_message].join("\n"))
+      end
+
+      def no_index_message
+        "No local index yet — run `portage index build` to give `find` a list of " \
+          "stores/products on top of stores.yml and web search."
+      end
 
-        Finding.new(check: "index", level: "info",
-                    message: "No local index yet — run `portage index build` to give `find` a list of " \
-                             "stores/products on top of stores.yml and web search.\n#{known_cache_message}")
+      def database_message(info)
+        "Index database: #{info[:path]} (#{info[:stores]} store row(s), #{info[:products]} product row(s), " \
+          "FTS5 #{info[:fts5] ? 'available' : 'not available'})."
       end
 
       def refresh_known_cache_if_stale

data/lib/portage/cli/find.rb

@@ -24,6 +24,12 @@ module Portage
     # search ranker and the purchase decision to `--yes` in one breath is how
     # you end up owning a counterfeit from a shop you've never heard of, so
     # picking a store stays an explicit act (see Cli.run_buy's `--store` gate).
+    #
+    # `store:` narrows the same pipeline to one store the caller already
+    # named — the live re-check of an index hit or an earlier offer. No
+    # search backend or OfferSource runs; the store is probed and its catalog
+    # searched, read-only, so its offers carry the same shape, `offer_ref`
+    # and history entry as any other find.
     class Find
       CART_CAP = "dev.ucp.shopping.cart".freeze
       CHECKOUT_CAP = "dev.ucp.shopping.checkout".freeze
@@ -43,8 +49,11 @@ module Portage
       #   candidate on it is never probed (see #call): it still surfaces as
       #   a candidate, marked `handoff_only: true`, so an agent can list it
       #   ("Amazon also sells this") without this process ever fetching it.
+      # @param store [String, nil] a store URL (see Find.store_origin): search
+      #   only that store, live, and skip every backend and OfferSource.
       def initialize(query:, limit: MAX_PROBES, max_price: nil, backends: nil, cache: nil, throttle: THROTTLE,
-                     offer_sources: nil, handoff_only: nil)
+                     offer_sources: nil, handoff_only: nil, store: nil)
+        @store = store
         @query = query.to_s
         @limit = [limit, MAX_PROBES].min
         @max_price = max_price
@@ -55,6 +64,27 @@ module Portage
         @handoff_only = handoff_only || HandoffOnly.new
       end
 
+      # The origin a `--store` value names: an http(s) URL (or a bare host,
+      # as `check` and `buy` accept) collapsed onto scheme://host[:port].
+      # @return [String, nil] nil for anything else, such as ftp:// or junk.
+      def self.store_origin(url)
+        text = url.to_s.strip
+        return nil if text.empty? || (text.include?("://") && !text.match?(%r{\Ahttps?://}i))
+
+        uri = URI.parse(text.include?("://") ? text : "https://#{text}")
+        origin_of_uri(uri)
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      def self.origin_of_uri(uri)
+        return nil unless uri.is_a?(URI::HTTP) && !uri.host.to_s.empty?
+
+        port = uri.port == uri.default_port ? "" : ":#{uri.port}"
+        "#{uri.scheme}://#{uri.host.downcase}#{port}"
+      end
+      private_class_method :origin_of_uri
+
       def call
         return report(message: "Nothing to search for — pass --query.") if @query.strip.empty?
 
@@ -88,6 +118,8 @@ module Portage
       # --- Step 1: ask the backends who might sell this ---
 
       def candidate_origins
+        return store_candidate if @store
+
         seen = {}
         @backends.each do |backend|
           urls_from(backend).each { |url| add_candidate(seen, backend, url) }
@@ -95,6 +127,15 @@ module Portage
         seen.values.first(@limit)
       end
 
+      # `--store`: the one candidate is the named store itself. A hand-off-only
+      # host stays a candidate, flagged, so #probe_candidates never fetches it.
+      def store_candidate
+        origin = self.class.store_origin(@store)
+        return [] unless origin
+
+        [{ origin: origin, source: "store", handoff_only: @handoff_only.host?(URI.parse(origin).host) }]
+      end
+
       # Keyed by host rather than by full origin: backends routinely hand back
       # both `http://` and `https://` for the same shop, and probing one host
       # twice over two schemes is a wasted request every time. https wins when
@@ -141,6 +182,8 @@ module Portage
       # #urls_from gives the URL backends. --max-price applies here exactly
       # as it does to a probed store's offers in #offer.
       def source_offers
+        return [] if @store
+
         offers = @offer_sources.flat_map do |source|
           source.offers(@query, limit: PER_STORE_RESULTS, context: BuyerContext.from_env)
         end
@@ -229,9 +272,11 @@ module Portage
         amount, currency = price_of(product)
         return nil if over_max_price?(amount)
 
-        { store: store[:origin], source: store[:source], checkout: store[:checkout],
-          product_id: field(product, "id"), title: field(product, "title"),
-          amount: amount, currency: currency, url: field(product, "url") }
+        OfferSources.with_product(
+          { store: store[:origin], source: store[:source], checkout: store[:checkout],
+            product_id: field(product, "id"), title: field(product, "title"),
+            amount: amount, currency: currency, url: field(product, "url") }, product
+        )
       end
 
       # An unpriced offer stays in: no price isn't the same as too dear.
@@ -279,6 +324,8 @@ module Portage
       end
 
       def no_candidates_message
+        return "#{@store.inspect} isn't an http(s) store URL." if @store
+
         names = @backends.map(&:name)
         return no_backends_message if names.empty?
 
@@ -310,6 +357,8 @@ module Portage
       end
 
       def summary(candidates, stores, offers)
+        return store_summary(candidates.first, stores, offers) if @store
+
         # Counted from the offers, not `stores`: an OfferSource's offers
         # come from stores that were never probed.
         selling = offers.map { |o| o[:store] }.uniq.length
@@ -318,6 +367,22 @@ module Portage
 
         "Checked #{candidates.length} store(s); none of them speak UCP."
       end
+
+      # The `--store` wording: say what happened to that one store, not "N
+      # stores".
+      def store_summary(candidate, stores, offers)
+        origin = candidate[:origin]
+        return "Found #{offers.length} offer(s) at #{origin}." if offers.any?
+        return handoff_only_message(origin) if candidate[:handoff_only]
+        return "#{origin} doesn't speak UCP, so there is no catalogue to search." if stores.empty?
+
+        "#{origin} speaks UCP but has nothing matching \"#{@query}\"."
+      end
+
+      def handoff_only_message(origin)
+        "#{origin} is a hand-off-only retailer: Portage never fetches it, so it wasn't searched. " \
+          "Open the site yourself."
+      end
     end
   end
 end

data/lib/portage/cli/index/builder.rb

@@ -37,6 +37,9 @@ module Portage
         THROTTLE = 0.1
         STALE_AFTER = 7 * 24 * 60 * 60
         TOP_CATEGORIES = 5
+        # Product sightings per ProductStore#upsert_many transaction — one
+        # products.json page's worth (docs/plans/local-catalogue.md Phase 2).
+        WRITE_BATCH = 250
 
         # Public so BrowserImport::Importer (Phase 3) labels a probed
         # origin's capabilities exactly the way an index build does.
@@ -94,13 +97,19 @@ module Portage
         # HandoffOnly) is recorded without ever probing it — the user
         # explicitly named it, but that's still not a request this process
         # sends.
-        def add(url)
+        #
+        # `crawl: true` (`index add URL --crawl`) then reads the store's own
+        # catalogue through Sources::StorefrontProducts into the index.
+        # Opt-in: a crawl is up to 21 more requests and 20s of pauses, where
+        # a plain add is one probe.
+        def add(url, crawl: false)
           origin = origin_of(url)
           return { added: false, message: "Not a valid http(s) URL: #{url}" } unless origin
           return store_manual_handoff_only(origin) if handoff_only_origin?(origin)
 
           session = probe(origin)
-          store_manual(origin, session)
+          result = store_manual(origin, session)
+          crawl ? crawl_added(origin, result) : result
         end
 
         # `portage index remove HOST`
@@ -184,10 +193,29 @@ module Portage
           { added: true, origin: origin, message: "Added #{origin} — hand-off only, never probed." }
         end
 
+        # `store_fields:` on a sighting (StorefrontProducts' crawl note and
+        # platform) lands on the store row as-is.
         def update_existing(origin, group)
           existing = @stores.find(origin)
+          fields = group.filter_map { |g| g[:store_fields] }.reduce({}, :merge)
           @stores.upsert(origin, sources: merged_sources(existing, group),
-                                 categories: merge_categories(existing["categories"], group))
+                                 categories: merge_categories(existing["categories"], group), **fields)
+        end
+
+        def crawl_added(origin, result)
+          sightings = Sources::StorefrontProducts.new(stores: @stores, handoff_only: @handoff_only)
+                                                 .crawl(origin, platform: @stores.find(origin)&.dig("platform"))
+          tagged = sightings.map { |s| s.merge(source: "storefront_products") }
+          apply(tagged, dry_run: false)
+          note = tagged.last.dig(:store_fields, :crawl)
+          result.merge(crawl: note, message: "#{result[:message]} #{crawl_message(note)}")
+        end
+
+        def crawl_message(note)
+          return "Catalogue not crawled (#{note['reason']})." if note["status"] == "skipped"
+
+          "Crawled #{note['products']} product(s) from #{note['pages']} page(s)" \
+            "#{" (stopped: #{note['reason']})" if note['reason']}."
         end
 
         def store_new(origin, session, group)
@@ -220,7 +248,7 @@ module Portage
           group.each do |sighting|
             next unless sighting[:title]
 
-            Classifier.categories_for(sighting[:title]).each { |id| tally[id] += 1 }
+            categories_of(sighting).each { |id| tally[id] += 1 }
           end
           tally.sort_by { |_id, weight| -weight }.first(TOP_CATEGORIES).to_h
         end
@@ -232,17 +260,26 @@ module Portage
         def capabilities_of(session) = self.class.capabilities_of(session)
 
         def store_products(sightings)
-          eligible = sightings.select { |s| s[:title] && @stores.find(s[:origin]) }
-          eligible.each { |sighting| store_product(sighting) }
+          known = Hash.new { |memo, origin| memo[origin] = !@stores.find(origin).nil? }
+          eligible = sightings.select { |s| s[:title] && known[s[:origin]] }
+          eligible.each_slice(WRITE_BATCH) { |batch| @products.upsert_many(batch.map { |s| product_row(s) }) }
           eligible.length
         end
 
-        def store_product(sighting)
-          key = product_key(sighting)
-          category = Classifier.categories_for(sighting[:title]).first
-          @products.upsert(key, origin: sighting[:origin], seen_at: @now.to_i, title: sighting[:title],
-                                brand: sighting[:brand], gtin: sighting[:gtin], category: category,
-                                sources: [sighting[:source]].compact)
+        # `product:` on a sighting (StorefrontProducts' handle, url,
+        # image_url, options, variant_ids) is stored alongside the usual
+        # fields.
+        def product_row(sighting)
+          { key: product_key(sighting), origin: sighting[:origin], seen_at: @now.to_i, title: sighting[:title],
+            brand: sighting[:brand], gtin: sighting[:gtin], category: categories_of(sighting).first,
+            sources: [sighting[:source]].compact, **sighting.fetch(:product, {}) }
+        end
+
+        # A source that already classified its sighting (StorefrontProducts,
+        # on product_type and tags) says so in `categories:`; otherwise the
+        # title is classified here.
+        def categories_of(sighting)
+          sighting[:categories] || Classifier.categories_for(sighting[:title])
         end
 
         # GTIN when a source has one (none do yet); otherwise a normalized