portage-cli 0.10.0 → 0.11.0

data/known-stores/category-stoplist.yml

@@ -0,0 +1,40 @@
+# Words Classifier never uses as evidence of a category, on either side: script/categories
+# leaves them out of categories.yml, and Classifier.categories_for drops them from its
+# input. Each entry is `word: why`. Both forms of a word are listed, because the stoplist is
+# matched exactly (the keyword match is what normalizes plurals).
+#
+# Two kinds of word belong here: words that describe how something is sold or filed rather
+# than what it is, and words that label a whole family of nodes ("accessories" names 76 of
+# the 192 level-2 nodes). A specific word that is merely ambiguous does not belong here.
+and: function word; appears in taxonomy names ("Hardware & Tools")
+for: function word; appears in taxonomy names ("Filters for ...")
+the: function word
+with: function word ("Vac with Multi-Purpose Head")
+from: function word
+new: merchandising flag; Light Yard tags products "New Collection"
+sale: merchandising flag; a sale tag says nothing about the product
+gift: merchandising flag; "Gift Set" is a way to sell, not a product type
+gifts: merchandising flag; plural of gift
+set: bundle word; "Gift Set" and "Furniture Set" say how it is packaged
+sets: bundle word; plural of set
+kit: bundle word; "drill kit" is the drill
+kits: bundle word; plural of kit
+collection: storefront and url word; matched "Toll Collection Devices" (4488) for every "New Collection" tag
+collections: url and storefront word; the /collections/ path segment of every Shopify collection url
+products: url word; the /products/ path segment of every Shopify product url
+product: url and page-title word
+category: url word; the /category/ path segment
+shop: page-title word ("Cordless Drills | Hardware | Shop")
+accessories: names 76 of the 192 level-2 nodes, so it cannot tell them apart
+accessory: singular of accessories
+supplies: names 25 level-2 nodes ("Pet Supplies", "Office Supplies"), so it cannot tell them apart
+supply: singular of supplies
+equipment: names 21 level-2 nodes, so it cannot tell them apart
+parts: names 21 level-2 nodes ("Vehicle Parts"), so it cannot tell them apart
+replacement: a spare, not a product type; "Replacement Fade Blade Set"
+general: filler word ("General Office Supplies")
+other: filler word; the catch-all name in many taxonomies
+miscellaneous: catch-all product type; JB Hi-Fi files 237 of its first 1,250 products under "MISCELLANEOUS"
+service: generic product-type word; JB Hi-Fi's "TELCO SERVICES" type matched Food Service (135)
+services: plural of service; JB Hi-Fi files 279 of its first 1,250 products under "TELCO SERVICES"
+hand: craft marker; Light Yard tags 146 of its 164 products "British Hand-Made", and "hand" also names 10 level-2 nodes ("Hand Tools")

data/known-stores/category-synonyms.yml

@@ -0,0 +1,15 @@
+# Words the Google taxonomy has no node for, added to a node's keywords by script/categories.
+# `id:` then `word: why`; the why names the golden case (spec/fixtures/classifier_golden.yml)
+# that needs it, by its text. Add a word here only for a case the golden set proves, never
+# in bulk.
+'594':
+  pendant: pendant light
+  sconce: wall sconce
+  bollard: Bollard light Bollard Lights British Hand-Made Driveway Lights Path Lights
+    Patio Lights Wooden Lights £250-£500
+'604':
+  whitegoods: WHITEGOODS Brand:Beko LimitedStock
+'262':
+  telco: TELCO SERVICES Brand:Samsung InStock
+'187':
+  boots: leather boots

data/lib/portage/cli/browser_import/categorize.rb

@@ -21,7 +21,8 @@ module Portage
         PRODUCT_PATH = %r{/products?/[^/?#]+}
 
         # @param rows [Array<Hash>] Readers rows, all for one domain.
-        # @return [Hash{String => Integer}] category id => weight, top 5.
+        # @return [Hash{String => Integer}] category id => weight, top 5 (equal
+        #   weights in the order first seen).
         def self.domain(rows)
           texts = Hash.new(0)
           rows.each { |row| texts[text_of(row)] += row[:visits] }
@@ -29,7 +30,9 @@ module Portage
           texts.each do |text, visits|
             Classifier.categories_for(text).each { |id| tally[id] += visits } unless text.empty?
           end
-          tally.sort_by { |_id, weight| -weight }.first(TOP_CATEGORIES).to_h
+          # `sort_by` isn't stable, so the position breaks ties: first seen wins.
+          tally.each_with_index.sort_by { |(_id, weight), seen| [-weight, seen] }.first(TOP_CATEGORIES)
+               .to_h { |pair, _seen| pair }
         end
 
         # @param kept [Array<Hash>] Importer's kept entries, rows included.

data/lib/portage/cli/check.rb

@@ -57,11 +57,21 @@ module Portage
         report = report.merge(adapter: adapter_for(report))
         report = report.merge(webmcp: webmcp_for(report))
         verdict = verdict_for(report)
-        report.merge(verdict: verdict, next_step: CheckNextStep.call(verdict, report))
+        with_index_hint(report.merge(verdict: verdict, next_step: CheckNextStep.call(verdict, report)))
       end
 
       private
 
+      # docs/plans/local-catalogue.md Phase 2: a Shopify (or native UCP)
+      # store's catalogue can be crawled into the local index. Check only
+      # names the command; it never crawls.
+      def with_index_hint(report)
+        return report unless report[:native_ucp] || report[:platform] == "Shopify"
+
+        port = @uri.port == @uri.default_port ? "" : ":#{@uri.port}"
+        report.merge(index_hint: "portage index add #{@uri.scheme}://#{@uri.host}#{port} --crawl")
+      end
+
       def handoff_only_report
         { url: @uri.to_s, native_ucp: nil, platform: nil, recommended_gem: nil, handoff_only: true, adapter: nil,
           webmcp: webmcp_skipped("#{@uri.host} is hand-off only, so it isn't probed"),

data/lib/portage/cli/classifier/ranking.rb

@@ -0,0 +1,135 @@
+module Portage
+  module Cli
+    module Classifier
+      # Scores the tokenized input against every node and keeps the best few
+      # (docs/plans/local-catalogue.md, Phase 5). Nodes carry hundreds of
+      # rolled-up keywords, so this is where "a generic word matches a dozen
+      # categories" gets dealt with.
+      module Ranking
+        # A node's own name words and its descendants' (`keywords`) say what
+        # the product is; its parent's (`parent_keywords`, e.g. "home",
+        # "garden") only where it is filed, so they count half as much.
+        KEYWORD_WEIGHT = 2
+        PARENT_WEIGHT = 1
+
+        # Callers treat every returned id as evidence (store routing, a
+        # browser domain's category tally), so the answer keeps only the best
+        # MAX_CATEGORIES ids and, of those, only ones scoring at least
+        # 1/CUTOFF of the best: "electric kettle" is Kitchen & Dining, not
+        # also Chairs because of "electric".
+        MAX_CATEGORIES = 3
+        CUTOFF = 2
+
+        module_function
+
+        # @param counts [Hash{String => Integer}] word => times the input says it.
+        # @param table [Classifier::Table]
+        # @param stopped [Hash] the stoplist.
+        # @return [Array<String>] up to MAX_CATEGORIES node ids, best first.
+        def best(counts, table, stopped)
+          ranked = score(counts, table).map do |id, (score, evidence)|
+            [id, score, evidence, *tie_breakers(table.nodes[id], counts.keys, stopped)]
+          end
+          ranked = ranked.sort_by { |(_id, score, _evidence, *rest)| [-score.round(6), *rest] }
+          cut(ranked).first(MAX_CATEGORIES).map(&:first)
+        end
+
+        # The first row always stays; later rows only if strong?.
+        def cut(ranked)
+          strongest = ranked.map { |row| row[2] }.max
+          ranked.each_with_index.select { |row, i| i.zero? || strong?(row[2], strongest) }.map(&:first)
+        end
+
+        # Distinct input words and how often each occurs. A plural variant of
+        # a word counts as the same word ("pendant" and "Pendants").
+        def word_counts(tokens)
+          tokens.each_with_object({}) do |token, counts|
+            word = counts.keys.find { |known| Classifier.word_match?(token, known) } || token
+            counts[word] = counts.fetch(word, 0) + 1
+          end
+        end
+
+        # @return [Hash{String => Array(Float, Float)}] node id => [score,
+        #   evidence], for nodes scoring above zero. Each word counts once for
+        #   a node, however many of its keywords match ("light" and "lights"
+        #   are two keywords but one word of the input), so a node cannot win
+        #   by listing a word's variants: KEYWORD_WEIGHT when a `keywords`
+        #   entry matches, PARENT_WEIGHT when only a `parent_keywords` entry
+        #   does, times how often the input says the word (1 + ln count: a
+        #   Shopify tag list repeats "Pendant Lights" once per room, which is
+        #   its best evidence, and the log keeps a long tag list from burying
+        #   a different word). `evidence` is the same sum with each word also
+        #   weighted by how rare it is (ln(1 + nodes / nodes it matches)), so
+        #   "kettle" counts for more than "electric".
+        def score(counts, table)
+          scores = Hash.new { |hash, id| hash[id] = [0.0, 0.0] }
+          counts.each do |word, count|
+            contributions(word, count, table).each { |id, weight, rarity| add(scores[id], weight, rarity) }
+          end
+          scores
+        end
+
+        # @return [Array<Array(String, Float, Float)>] [node id, weight,
+        #   rarity] for every node `word` matches.
+        def contributions(word, count, table)
+          own = matching_ids(word, table.own)
+          parent = matching_ids(word, table.parent) - own
+          return [] if own.empty? && parent.empty?
+
+          tf = 1 + Math.log(count)
+          rarity = Math.log(1 + table.nodes.size.fdiv((own + parent).length))
+          own.map { |id| [id, KEYWORD_WEIGHT * tf, rarity] } + parent.map { |id| [id, PARENT_WEIGHT * tf, rarity] }
+        end
+
+        def add(pair, weight, rarity)
+          pair[0] += weight
+          pair[1] += weight * rarity
+        end
+
+        # Ranking by `score` alone is what classifies a long tag list best,
+        # because rare words are mostly noise there (a room name in a
+        # lighting store's tags). The cut uses `evidence`, which a generic
+        # word cannot fill: an id beyond the first stays only if its evidence
+        # reaches 1/CUTOFF of the strongest.
+        def strong?(evidence, strongest) = (evidence * CUTOFF) - strongest > -1e-9
+
+        # What separates nodes with equal scores, best first: the share of
+        # the node's own name the input covers ("Sofas" before "Sofa
+        # Accessories" for "sofa"), then a name with no stoplisted word in it
+        # ("Household Appliances" before "Household Appliance Accessories"),
+        # then fewer keywords (the smaller node is the more specific one),
+        # then the file's own order.
+        def tie_breakers(node, words, stopped)
+          name = node["name"].to_s.split(" > ").last.to_s.downcase.split(/[^\p{Alpha}]+/)
+                             .select { |word| word.length >= Classifier::MIN_WORD_LENGTH }
+          covered = name.count { |part| words.any? { |word| Classifier.word_match?(word, part) } }
+          generic = name.any? { |part| stopped.key?(part) } ? 1 : 0
+          [-covered.fdiv([name.length, 1].max), generic, Array(node["keywords"]).length, node["order"]]
+        end
+
+        # @return [Array<String>] ids of the nodes in `index` with a keyword
+        #   `word` matches.
+        def matching_ids(word, index)
+          keywords_matching(word).flat_map { |keyword| index.fetch(keyword, []) }.uniq
+        end
+
+        # The keywords Classifier.word_match? accepts `word` for: itself, the
+        # singular it is a plural of ("boots" -> "boot", "watches" ->
+        # "watch"), the plural of it ("boot" -> "boots"), and the "y"/"ies"
+        # swap. Looking a word's few candidate keywords up in a hash, rather
+        # than comparing every word with every keyword, keeps a long
+        # product-tag text (Index::Sources::StorefrontProducts,
+        # docs/plans/local-catalogue.md Phase 2) cheap now that nodes carry
+        # hundreds of keywords.
+        def keywords_matching(word)
+          keywords = [word, "#{word}s", "#{word}es"]
+          keywords << word.delete_suffix("s") if word.end_with?("s")
+          keywords << word.delete_suffix("es") if word.end_with?("es")
+          keywords << "#{word[0..-4]}y" if word.end_with?("ies")
+          keywords << "#{word[0..-2]}ies" if word.end_with?("y")
+          keywords
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/classifier/table.rb

@@ -0,0 +1,63 @@
+require "yaml"
+
+module Portage
+  module Cli
+    module Classifier
+      # The shipped categories.yml and the user's ~/.portage/categories.yml,
+      # merged (a user node replaces a shipped node of the same id), with the
+      # keyword => node ids indexes the scoring looks words up in.
+      Table = Struct.new(:nodes, :own, :parent)
+
+      class Table
+        @cache = {}
+
+        class << self
+          # Built once per pair of files and kept for the process, keyed by
+          # the files' mtime and size, so an edit to ~/.portage/categories.yml
+          # still shows up on the next call; classifying a whole catalogue
+          # would otherwise rebuild the index for every product.
+          def for(known_path, user_path)
+            key = [signature(known_path), signature(user_path)]
+            @cache.clear if @cache.size > 8
+            @cache[key] ||= build(known_path, user_path)
+          end
+
+          # @return [Hash] the YAML mapping at `path`; empty when the file is
+          #   missing, unreadable or not a mapping.
+          def load_yaml(path)
+            return {} unless path && File.readable?(path)
+
+            data = YAML.safe_load_file(path)
+            data.is_a?(Hash) ? data : {}
+          rescue StandardError
+            {}
+          end
+
+          private
+
+          def build(known_path, user_path)
+            ordered = {}
+            load_yaml(known_path).each_with_index { |(id, node), i| ordered[id] = node.merge("order" => i) }
+            load_yaml(user_path).each_with_index do |(id, node), i|
+              ordered[id] = node.merge("order" => ordered.size + i)
+            end
+            new(ordered, index(ordered, "keywords"), index(ordered, "parent_keywords"))
+          end
+
+          def index(nodes, field)
+            result = Hash.new { |hash, key| hash[key] = [] }
+            nodes.each { |id, node| Array(node[field]).each { |keyword| result[keyword] << id } }
+            result.default_proc = nil
+            result
+          end
+
+          def signature(path)
+            return nil unless path && File.readable?(path)
+
+            [path, File.mtime(path).to_r, File.size(path)]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/classifier.rb

@@ -1,4 +1,6 @@
 require "yaml"
+require_relative "classifier/table"
+require_relative "classifier/ranking"
 
 module Portage
   module Cli
@@ -18,10 +20,21 @@ module Portage
     # check matches in both directions ("carpet" contains "pet", "chair"
     # contains "hair", "scarf" contains "car") and was a real source of
     # false positives before this became whole-word.
+    #
+    # `known-stores/categories.yml` is generated by `script/categories`: a level-2
+    # node's `keywords` are its own name plus its descendants', `parent_keywords`
+    # are its parent's words, and `category-stoplist.yml` words are left out.
+    # See Ranking for how a text is scored against it.
     module Classifier
       # `known-stores/categories.yml`, from `lib/portage/cli/classifier.rb`.
       KNOWN_PATH = File.expand_path("../../../known-stores/categories.yml", __dir__).freeze
 
+      # Words that are never evidence of a category (`known-stores/category-stoplist.yml`,
+      # `word: why`): merchandising and url words, and words that name a whole family of
+      # nodes. `script/categories` leaves them out of the keywords; `categories_for`
+      # drops them from its input, so the two sides agree.
+      STOPLIST_PATH = File.expand_path("../../../known-stores/category-stoplist.yml", __dir__).freeze
+
       # The user's own additions/overrides — same id overrides a shipped
       # node's keywords, a new id extends the taxonomy. Absent by default;
       # nothing here is required for the shipped file to work.
@@ -44,21 +57,22 @@ module Portage
       # @param known_path [String] override for KNOWN_PATH — specs redirect
       #   this the same way SearchBackends::Allowlist takes its own `path:`.
       # @param user_path [String] override for PATH.
-      # @return [Array<String>] category ids, most keyword hits first. Ties
-      #   keep the shipped file's own order. Empty when nothing matches.
-      def self.categories_for(text, known_path: KNOWN_PATH, user_path: PATH)
-        words = tokenize(text)
-        return [] if words.empty?
-
-        scored = nodes(known_path, user_path).filter_map { |id, node| rank(id, node, words) }
-        scored.sort_by { |(_id, score, order)| [-score, order] }.map(&:first)
+      # @param stoplist_path [String] override for STOPLIST_PATH.
+      # @return [Array<String>] up to MAX_CATEGORIES category ids, best score
+      #   first (ties: see .tie_breakers). Empty when nothing matches.
+      def self.categories_for(text, known_path: KNOWN_PATH, user_path: PATH, stoplist_path: STOPLIST_PATH)
+        stopped = Table.load_yaml(stoplist_path)
+        counts = Ranking.word_counts(tokenize(text).reject { |word| stopped.key?(word) })
+        return [] if counts.empty?
+
+        Ranking.best(counts, Table.for(known_path, user_path), stopped)
       end
 
       # @return [Array<String>] the taxonomy names for `ids`, in order —
       #   `portage browser import` (Phase 3) shows a domain's guessed
       #   categories by name so the user can judge them before saving.
       def self.names_for(ids, known_path: KNOWN_PATH, user_path: PATH)
-        all = nodes(known_path, user_path)
+        all = Table.for(known_path, user_path).nodes
         Array(ids).filter_map { |id| all.dig(id.to_s, "name") }
       end
 
@@ -88,17 +102,6 @@ module Portage
                                                     .select { |word| word.length >= MIN_WORD_LENGTH }
       end
 
-      # --- Scoring one node against the tokenized input ---
-
-      def self.rank(id, node, words)
-        keywords = Array(node["keywords"])
-        hits = keywords.count { |keyword| words.any? { |word| word_match?(word, keyword) } }
-        return nil unless hits.positive?
-
-        [id, hits, node["order"].to_i]
-      end
-      private_class_method :rank
-
       # Whole-word only, plus the plural forms a keyword list and a real
       # query/title actually differ by: an exact match, one plus a trailing
       # "s" or "es" ("boot"/"boots", "watch"/"watches"), or the "y"/"ies"
@@ -129,30 +132,6 @@ module Portage
           (keyword.end_with?("ies") && word == "#{keyword[0..-4]}y")
       end
       private_class_method :ies_y_match?
-
-      # --- Loading and merging the two files ---
-
-      # Re-read on every call rather than cached process-wide: `find` calls
-      # this once or twice per invocation, not in a hot loop, and a cached
-      # copy would miss an edit to ~/.portage/categories.yml until the next
-      # process start.
-      def self.nodes(known_path, user_path)
-        ordered = {}
-        load_yaml(known_path).each_with_index { |(id, node), i| ordered[id] = node.merge("order" => i) }
-        load_yaml(user_path).each_with_index { |(id, node), i| ordered[id] = node.merge("order" => ordered.size + i) }
-        ordered
-      end
-      private_class_method :nodes
-
-      def self.load_yaml(path)
-        return {} unless path && File.readable?(path)
-
-        data = YAML.safe_load_file(path)
-        data.is_a?(Hash) ? data : {}
-      rescue StandardError
-        {}
-      end
-      private_class_method :load_yaml
     end
   end
 end

data/lib/portage/cli/doctor.rb

@@ -42,7 +42,8 @@ module Portage
       #   those four warnings were noise on every fresh install.
       def initialize(adapter_class: nil, proxy_settings: ProxySettings.new, install_doctor: InstallDoctor.new,
                      seller: true, dot_env_path: DotEnv.loaded_path, index_stores: Index::Store.new,
-                     index_products: Index::ProductStore.new, known_cache: Index::KnownCache.new)
+                     index_products: Index::ProductStore.new, known_cache: Index::KnownCache.new,
+                     index_database: Index::Database.new(path: Index::Database.path_for(Index::Store::PATH)))
         @adapter_class = adapter_class
         @proxy_settings = proxy_settings
         @install_doctor = install_doctor
@@ -51,6 +52,7 @@ module Portage
         @index_stores = index_stores
         @index_products = index_products
         @known_cache = known_cache
+        @index_database = index_database
       end
 
       # `portage setup` always offers its wizard on a TTY; a bare `portage
@@ -210,12 +212,20 @@ module Portage
       # leaves behind.
       def index_finding
         refresh_known_cache_if_stale
-        return Finding.new(check: "index", level: "info", message: "#{index_message}\n#{known_cache_message}") \
-          if @index_stores.exists?
+        first = @index_stores.exists? ? index_message : no_index_message
+        database = @index_database.info
+        Finding.new(check: "index", level: "info", details: { database: database },
+                    message: [first, database_message(database), known_cache_message].join("\n"))
+      end
+
+      def no_index_message
+        "No local index yet — run `portage index build` to give `find` a list of " \
+          "stores/products on top of stores.yml and web search."
+      end
 
-        Finding.new(check: "index", level: "info",
-                    message: "No local index yet — run `portage index build` to give `find` a list of " \
-                             "stores/products on top of stores.yml and web search.\n#{known_cache_message}")
+      def database_message(info)
+        "Index database: #{info[:path]} (#{info[:stores]} store row(s), #{info[:products]} product row(s), " \
+          "FTS5 #{info[:fts5] ? 'available' : 'not available'})."
       end
 
       def refresh_known_cache_if_stale

data/lib/portage/cli/find.rb

@@ -229,9 +229,11 @@ module Portage
         amount, currency = price_of(product)
         return nil if over_max_price?(amount)
 
-        { store: store[:origin], source: store[:source], checkout: store[:checkout],
-          product_id: field(product, "id"), title: field(product, "title"),
-          amount: amount, currency: currency, url: field(product, "url") }
+        OfferSources.with_product(
+          { store: store[:origin], source: store[:source], checkout: store[:checkout],
+            product_id: field(product, "id"), title: field(product, "title"),
+            amount: amount, currency: currency, url: field(product, "url") }, product
+        )
       end
 
       # An unpriced offer stays in: no price isn't the same as too dear.

data/lib/portage/cli/index/builder.rb

@@ -37,6 +37,9 @@ module Portage
         THROTTLE = 0.1
         STALE_AFTER = 7 * 24 * 60 * 60
         TOP_CATEGORIES = 5
+        # Product sightings per ProductStore#upsert_many transaction — one
+        # products.json page's worth (docs/plans/local-catalogue.md Phase 2).
+        WRITE_BATCH = 250
 
         # Public so BrowserImport::Importer (Phase 3) labels a probed
         # origin's capabilities exactly the way an index build does.
@@ -94,13 +97,19 @@ module Portage
         # HandoffOnly) is recorded without ever probing it — the user
         # explicitly named it, but that's still not a request this process
         # sends.
-        def add(url)
+        #
+        # `crawl: true` (`index add URL --crawl`) then reads the store's own
+        # catalogue through Sources::StorefrontProducts into the index.
+        # Opt-in: a crawl is up to 21 more requests and 20s of pauses, where
+        # a plain add is one probe.
+        def add(url, crawl: false)
           origin = origin_of(url)
           return { added: false, message: "Not a valid http(s) URL: #{url}" } unless origin
           return store_manual_handoff_only(origin) if handoff_only_origin?(origin)
 
           session = probe(origin)
-          store_manual(origin, session)
+          result = store_manual(origin, session)
+          crawl ? crawl_added(origin, result) : result
         end
 
         # `portage index remove HOST`
@@ -184,10 +193,29 @@ module Portage
           { added: true, origin: origin, message: "Added #{origin} — hand-off only, never probed." }
         end
 
+        # `store_fields:` on a sighting (StorefrontProducts' crawl note and
+        # platform) lands on the store row as-is.
         def update_existing(origin, group)
           existing = @stores.find(origin)
+          fields = group.filter_map { |g| g[:store_fields] }.reduce({}, :merge)
           @stores.upsert(origin, sources: merged_sources(existing, group),
-                                 categories: merge_categories(existing["categories"], group))
+                                 categories: merge_categories(existing["categories"], group), **fields)
+        end
+
+        def crawl_added(origin, result)
+          sightings = Sources::StorefrontProducts.new(stores: @stores, handoff_only: @handoff_only)
+                                                 .crawl(origin, platform: @stores.find(origin)&.dig("platform"))
+          tagged = sightings.map { |s| s.merge(source: "storefront_products") }
+          apply(tagged, dry_run: false)
+          note = tagged.last.dig(:store_fields, :crawl)
+          result.merge(crawl: note, message: "#{result[:message]} #{crawl_message(note)}")
+        end
+
+        def crawl_message(note)
+          return "Catalogue not crawled (#{note['reason']})." if note["status"] == "skipped"
+
+          "Crawled #{note['products']} product(s) from #{note['pages']} page(s)" \
+            "#{" (stopped: #{note['reason']})" if note['reason']}."
         end
 
         def store_new(origin, session, group)
@@ -220,7 +248,7 @@ module Portage
           group.each do |sighting|
             next unless sighting[:title]
 
-            Classifier.categories_for(sighting[:title]).each { |id| tally[id] += 1 }
+            categories_of(sighting).each { |id| tally[id] += 1 }
           end
           tally.sort_by { |_id, weight| -weight }.first(TOP_CATEGORIES).to_h
         end
@@ -232,17 +260,26 @@ module Portage
         def capabilities_of(session) = self.class.capabilities_of(session)
 
         def store_products(sightings)
-          eligible = sightings.select { |s| s[:title] && @stores.find(s[:origin]) }
-          eligible.each { |sighting| store_product(sighting) }
+          known = Hash.new { |memo, origin| memo[origin] = !@stores.find(origin).nil? }
+          eligible = sightings.select { |s| s[:title] && known[s[:origin]] }
+          eligible.each_slice(WRITE_BATCH) { |batch| @products.upsert_many(batch.map { |s| product_row(s) }) }
           eligible.length
         end
 
-        def store_product(sighting)
-          key = product_key(sighting)
-          category = Classifier.categories_for(sighting[:title]).first
-          @products.upsert(key, origin: sighting[:origin], seen_at: @now.to_i, title: sighting[:title],
-                                brand: sighting[:brand], gtin: sighting[:gtin], category: category,
-                                sources: [sighting[:source]].compact)
+        # `product:` on a sighting (StorefrontProducts' handle, url,
+        # image_url, options, variant_ids) is stored alongside the usual
+        # fields.
+        def product_row(sighting)
+          { key: product_key(sighting), origin: sighting[:origin], seen_at: @now.to_i, title: sighting[:title],
+            brand: sighting[:brand], gtin: sighting[:gtin], category: categories_of(sighting).first,
+            sources: [sighting[:source]].compact, **sighting.fetch(:product, {}) }
+        end
+
+        # A source that already classified its sighting (StorefrontProducts,
+        # on product_type and tags) says so in `categories:`; otherwise the
+        # title is classified here.
+        def categories_of(sighting)
+          sighting[:categories] || Classifier.categories_for(sighting[:title])
         end
 
         # GTIN when a source has one (none do yet); otherwise a normalized