portage-cli 0.9.0 → 0.11.0

data/known-stores/category-stoplist.yml

@@ -0,0 +1,40 @@
+# Words Classifier never uses as evidence of a category, on either side: script/categories
+# leaves them out of categories.yml, and Classifier.categories_for drops them from its
+# input. Each entry is `word: why`. Both forms of a word are listed, because the stoplist is
+# matched exactly (the keyword match is what normalizes plurals).
+#
+# Two kinds of word belong here: words that describe how something is sold or filed rather
+# than what it is, and words that label a whole family of nodes ("accessories" names 76 of
+# the 192 level-2 nodes). A specific word that is merely ambiguous does not belong here.
+and: function word; appears in taxonomy names ("Hardware & Tools")
+for: function word; appears in taxonomy names ("Filters for ...")
+the: function word
+with: function word ("Vac with Multi-Purpose Head")
+from: function word
+new: merchandising flag; Light Yard tags products "New Collection"
+sale: merchandising flag; a sale tag says nothing about the product
+gift: merchandising flag; "Gift Set" is a way to sell, not a product type
+gifts: merchandising flag; plural of gift
+set: bundle word; "Gift Set" and "Furniture Set" say how it is packaged
+sets: bundle word; plural of set
+kit: bundle word; "drill kit" is the drill
+kits: bundle word; plural of kit
+collection: storefront and url word; matched "Toll Collection Devices" (4488) for every "New Collection" tag
+collections: url and storefront word; the /collections/ path segment of every Shopify collection url
+products: url word; the /products/ path segment of every Shopify product url
+product: url and page-title word
+category: url word; the /category/ path segment
+shop: page-title word ("Cordless Drills | Hardware | Shop")
+accessories: names 76 of the 192 level-2 nodes, so it cannot tell them apart
+accessory: singular of accessories
+supplies: names 25 level-2 nodes ("Pet Supplies", "Office Supplies"), so it cannot tell them apart
+supply: singular of supplies
+equipment: names 21 level-2 nodes, so it cannot tell them apart
+parts: names 21 level-2 nodes ("Vehicle Parts"), so it cannot tell them apart
+replacement: a spare, not a product type; "Replacement Fade Blade Set"
+general: filler word ("General Office Supplies")
+other: filler word; the catch-all name in many taxonomies
+miscellaneous: catch-all product type; JB Hi-Fi files 237 of its first 1,250 products under "MISCELLANEOUS"
+service: generic product-type word; JB Hi-Fi's "TELCO SERVICES" type matched Food Service (135)
+services: plural of service; JB Hi-Fi files 279 of its first 1,250 products under "TELCO SERVICES"
+hand: craft marker; Light Yard tags 146 of its 164 products "British Hand-Made", and "hand" also names 10 level-2 nodes ("Hand Tools")

data/known-stores/category-synonyms.yml

@@ -0,0 +1,15 @@
+# Words the Google taxonomy has no node for, added to a node's keywords by script/categories.
+# `id:` then `word: why`; the why names the golden case (spec/fixtures/classifier_golden.yml)
+# that needs it, by its text. Add a word here only for a case the golden set proves, never
+# in bulk.
+'594':
+  pendant: pendant light
+  sconce: wall sconce
+  bollard: Bollard light Bollard Lights British Hand-Made Driveway Lights Path Lights
+    Patio Lights Wooden Lights £250-£500
+'604':
+  whitegoods: WHITEGOODS Brand:Beko LimitedStock
+'262':
+  telco: TELCO SERVICES Brand:Samsung InStock
+'187':
+  boots: leather boots

data/lib/portage/cli/browser_import/categorize.rb

@@ -21,7 +21,8 @@ module Portage
         PRODUCT_PATH = %r{/products?/[^/?#]+}
 
         # @param rows [Array<Hash>] Readers rows, all for one domain.
-        # @return [Hash{String => Integer}] category id => weight, top 5.
+        # @return [Hash{String => Integer}] category id => weight, top 5 (equal
+        #   weights in the order first seen).
         def self.domain(rows)
           texts = Hash.new(0)
           rows.each { |row| texts[text_of(row)] += row[:visits] }
@@ -29,7 +30,9 @@ module Portage
           texts.each do |text, visits|
             Classifier.categories_for(text).each { |id| tally[id] += visits } unless text.empty?
           end
-          tally.sort_by { |_id, weight| -weight }.first(TOP_CATEGORIES).to_h
+          # `sort_by` isn't stable, so the position breaks ties: first seen wins.
+          tally.each_with_index.sort_by { |(_id, weight), seen| [-weight, seen] }.first(TOP_CATEGORIES)
+               .to_h { |pair, _seen| pair }
         end
 
         # @param kept [Array<Hash>] Importer's kept entries, rows included.

data/lib/portage/cli/buy.rb

@@ -13,6 +13,7 @@ require_relative "checkout_handoff"
 require_relative "money"
 require_relative "notifier"
 require_relative "handoff_only"
+require_relative "handoff_host"
 require_relative "offer_sources"
 require_relative "handoff_target"
 require_relative "handoff_agents"
@@ -189,32 +190,10 @@ module Portage
       # "handoff_only_hosts" is the user's own list (see HandoffOnly).
       def handoff_only?
         @handoff_only ||= HandoffOnly.new
-        return true if @handoff_only.host?(@uri.host)
-        return true if OfferSources.retail_handoff_host?(@uri.host)
-
-        etsy_buyer_host?
-      end
-
-      # Phase 7 (docs/plans/buy-skill-and-local-browser.md):
-      # portage-ucp-etsy is a *seller*-side adapter — a shop owner with
-      # their own ETSY_* credentials set still reaches #adapter_flow
-      # unchanged below, the same "only your own store" rule every other
-      # adapter already gets (see the class comment at the top of this
-      # file). An ordinary buyer with no Etsy credentials of their own
-      # gets routed to hand-off here instead of a homepage fetch +
-      # platform detection that would just be scraping a stranger's
-      # listing — the buyer-side offer OfferSources::EtsyListings hands
-      # `find` has nothing this process can check out anyway.
-      def etsy_buyer_host?
-        HandoffOnly.matches_any?(@uri.host, %w[etsy.com]) && !etsy_adapter_configured?
+        HandoffHost.restricted?(@uri.host, handoff_only: @handoff_only)
       end
 
-      def etsy_adapter_configured?
-        platform = Portage::Ucp::Resolver::PLATFORMS.find { |p| p.name == "Etsy" }
-        return false unless platform
-
-        Portage::Ucp::Resolver.missing_env(platform, Portage::Ucp::Resolver.env_for(platform)).empty?
-      end
+      def etsy_buyer_host? = HandoffHost.etsy_buyer_host?(@uri.host)
 
       # No checkout was ever built — there's nothing to browse or complete,
       # just a link and why. `checkout_url` is built, never fetched: the

data/lib/portage/cli/check.rb

@@ -0,0 +1,164 @@
+require "uri"
+require "portage/ucp"
+require_relative "handoff_only"
+require_relative "handoff_host"
+require_relative "check_next_step"
+require_relative "webmcp"
+require_relative "browser_profile"
+
+module Portage
+  module Cli
+    # `portage check <url>` — "can Portage buy from this store, and how?"
+    #
+    # Wraps Portage::Ucp::Check (native `/.well-known/ucp`, platform
+    # detection, live adapter probe) and adds what only the CLI knows, so
+    # the answer matches what `portage buy` would actually do: the
+    # hand-off-only list (HandoffHost, shared with Buy), whether the adapter
+    # gem is installed, and whether the page exposes WebMCP tools.
+    #
+    # Precedence mirrors Buy#call: hand-off-only host, then native UCP, then
+    # WebMCP, then a platform adapter, then nothing. A hand-off-only host
+    # short-circuits before any request is made.
+    #
+    # Read-only and quiet: plain GETs only (through Ucp::Check), never a
+    # cart, never a checkout. WebMCP needs a live page, and Portage never
+    # launches a browser for this: it only reads tools from a tab the
+    # Portage browser profile already has open on the store's host
+    # (`portage browser profile open --url ...`). With no such tab, or with
+    # portage-ucp-webmcp not installed, `webmcp.status` is "skipped" and says
+    # why. A `webmcp_bridge:` can be injected instead (specs, library use).
+    class Check
+      CART_CAP = "dev.ucp.shopping.cart".freeze
+      CHECKOUT_CAP = "dev.ucp.shopping.checkout".freeze
+
+      # Verdicts `portage check` exits 0 for.
+      USABLE_VERDICTS = %w[automated webmcp].freeze
+
+      NO_TAB_REASON = "no Portage browser profile tab is open on this store " \
+                      "(portage browser profile open --url URL), and check never launches a browser".freeze
+
+      def self.call(url, **) = new(url, **).call
+
+      # @param webmcp_bridge [#list_tools, nil] a bridge over the store's page.
+      # @param handoff_only [HandoffOnly]
+      # @param checker [#call] url -> Portage::Ucp::Check report (injectable).
+      def initialize(url, webmcp_bridge: nil, handoff_only: HandoffOnly.new, checker: Portage::Ucp::Check.method(:call))
+        raw = url.to_s.strip
+        @uri = URI.parse(raw =~ %r{\Ahttps?://}i ? raw : "https://#{raw}")
+        @webmcp_bridge = webmcp_bridge
+        @handoff_only = handoff_only
+        @checker = checker
+      end
+
+      def call
+        return handoff_only_report if HandoffHost.restricted?(@uri.host, handoff_only: @handoff_only)
+
+        report = @checker.call(@uri.to_s).merge(handoff_only: false)
+        report = report.merge(adapter: adapter_for(report))
+        report = report.merge(webmcp: webmcp_for(report))
+        verdict = verdict_for(report)
+        with_index_hint(report.merge(verdict: verdict, next_step: CheckNextStep.call(verdict, report)))
+      end
+
+      private
+
+      # docs/plans/local-catalogue.md Phase 2: a Shopify (or native UCP)
+      # store's catalogue can be crawled into the local index. Check only
+      # names the command; it never crawls.
+      def with_index_hint(report)
+        return report unless report[:native_ucp] || report[:platform] == "Shopify"
+
+        port = @uri.port == @uri.default_port ? "" : ":#{@uri.port}"
+        report.merge(index_hint: "portage index add #{@uri.scheme}://#{@uri.host}#{port} --crawl")
+      end
+
+      def handoff_only_report
+        { url: @uri.to_s, native_ucp: nil, platform: nil, recommended_gem: nil, handoff_only: true, adapter: nil,
+          webmcp: webmcp_skipped("#{@uri.host} is hand-off only, so it isn't probed"),
+          verdict: "handoff",
+          next_step: "#{@uri.host} restricts automated purchasing agents. Portage opens the page and you buy." }
+      end
+
+      # --- adapter ---
+
+      def adapter_for(report)
+        platform = Portage::Ucp::Resolver::PLATFORMS.find { |p| p.name == report[:platform] }
+        return nil unless platform
+
+        { gem: platform.gem, installed: gem_installed?(platform),
+          missing_env: Portage::Ucp::Resolver.missing_env(platform, Portage::Ucp::Resolver.env_for(platform)) }
+      end
+
+      # Same test the live probe uses: the adapter's require_path either
+      # loads or it doesn't. Loading has no I/O.
+      def gem_installed?(platform)
+        require platform.require_path
+        true
+      rescue LoadError
+        false
+      end
+
+      # --- webmcp ---
+
+      def webmcp_for(report)
+        return webmcp_skipped("the store speaks UCP natively") if report[:native_ucp]
+        return webmcp_skipped("portage-ucp-webmcp isn't installed") unless Webmcp.available?
+
+        bridge = @webmcp_bridge || existing_tab_bridge
+        return webmcp_skipped(NO_TAB_REASON) unless bridge
+
+        detect_webmcp(bridge)
+      rescue StandardError => e
+        { status: "error", tools: [], reason: "#{e.class}: #{e.message}" }
+      end
+
+      def webmcp_skipped(reason) = { status: "skipped", tools: [], reason: reason }
+
+      def detect_webmcp(bridge)
+        tools = bridge.list_tools
+        names = tools.map { |tool| (tool["name"] || tool[:name]).to_s }
+        return { status: "none", tools: [], reason: "the page registers no WebMCP tools" } if names.empty?
+
+        preset = Portage::Ucp::WebMcp::Presets.detect(tools)
+        session = Portage::Ucp::WebMcp.connect(bridge: bridge, preset: preset)
+        return { status: "available", tools: names, reason: nil } if cart_and_checkout?(session)
+
+        { status: "none", tools: names,
+          reason: "the page's WebMCP tools aren't a cart and checkout set Portage recognises" }
+      end
+
+      def cart_and_checkout?(session) = session.advertises?(CART_CAP) && session.advertises?(CHECKOUT_CAP)
+
+      # Only ever a tab that is already open on the store's host: no new tab
+      # is created, no browser is started, nothing is navigated.
+      def existing_tab_bridge
+        profile = BrowserProfile::Profile.new
+        return nil unless profile.status[:running]
+
+        tab = BrowserProfile::Cdp.list(port: profile.port).find { |t| t["type"] == "page" && same_host?(t["url"]) }
+        ws_url = tab && tab["webSocketDebuggerUrl"]
+        return nil unless ws_url
+
+        BrowserProfile::Bridge.new(socket: BrowserProfile::CdpSocket.connect(ws_url),
+                                   allowlist: BrowserProfile::Allowlist.new(hosts: [@uri.host]))
+      end
+
+      def same_host?(url)
+        URI(url.to_s).host == @uri.host
+      rescue URI::InvalidURIError
+        false
+      end
+
+      # --- verdict ---
+
+      def verdict_for(report)
+        return "automated" if report[:native_ucp]
+        return "webmcp" if report.dig(:webmcp, :status) == "available"
+        return "automated" if report.dig(:live_probe, :status) == "ok"
+        return "handoff" if report[:platform]
+
+        "unsupported"
+      end
+    end
+  end
+end

data/lib/portage/cli/check_next_step.rb

@@ -0,0 +1,51 @@
+module Portage
+  module Cli
+    # The plain-English `next_step` of a Check report, one sentence per
+    # verdict — kept apart from Check so its detection logic stays readable.
+    module CheckNextStep
+      def self.call(verdict, report)
+        case verdict
+        when "automated" then automated(report)
+        when "webmcp" then "Portage will build the cart; you pay in your browser."
+        when "handoff" then adapter(report)
+        else unsupported(report)
+        end
+      end
+
+      def self.automated(report)
+        return "Nothing to do — this store speaks UCP natively." if report[:native_ucp]
+
+        "Nothing more to set up — the #{report[:platform]} adapter (#{report.dig(:adapter, :gem)}) is installed " \
+          "and answered a live probe."
+      end
+
+      def self.adapter(report)
+        adapter = report[:adapter]
+        return "#{report[:url]} restricts automated purchasing agents. Portage opens the page and you buy." unless
+          adapter
+
+        actions = []
+        actions << "install #{adapter[:gem]}" unless adapter[:installed]
+        actions << "set #{adapter[:missing_env].join(', ')}" if adapter[:missing_env].any?
+        return probe_failed(report) if actions.empty?
+
+        "To automate this #{report[:platform]} store (adapters act as the store's owner), " \
+          "#{actions.join(' and ')}. Until then Portage opens the store and you buy."
+      end
+
+      def self.probe_failed(report)
+        reason = report.dig(:live_probe, :reason)
+        "The #{report[:platform]} adapter is set up but its live probe failed#{" (#{reason})" if reason}. " \
+          "Run `portage doctor`; until then Portage opens the store and you buy."
+      end
+
+      def self.unsupported(report)
+        webmcp = report[:webmcp]
+        note = webmcp[:status] == "skipped" ? " (WebMCP not checked: #{webmcp[:reason]})" : ""
+        "No UCP manifest, known platform or WebMCP cart found#{note}. Portage can only open the store " \
+          "for you to browse."
+      end
+      private_class_method :automated, :adapter, :probe_failed, :unsupported
+    end
+  end
+end

data/lib/portage/cli/classifier/ranking.rb

@@ -0,0 +1,135 @@
+module Portage
+  module Cli
+    module Classifier
+      # Scores the tokenized input against every node and keeps the best few
+      # (docs/plans/local-catalogue.md, Phase 5). Nodes carry hundreds of
+      # rolled-up keywords, so this is where "a generic word matches a dozen
+      # categories" gets dealt with.
+      module Ranking
+        # A node's own name words and its descendants' (`keywords`) say what
+        # the product is; its parent's (`parent_keywords`, e.g. "home",
+        # "garden") only where it is filed, so they count half as much.
+        KEYWORD_WEIGHT = 2
+        PARENT_WEIGHT = 1
+
+        # Callers treat every returned id as evidence (store routing, a
+        # browser domain's category tally), so the answer keeps only the best
+        # MAX_CATEGORIES ids and, of those, only ones scoring at least
+        # 1/CUTOFF of the best: "electric kettle" is Kitchen & Dining, not
+        # also Chairs because of "electric".
+        MAX_CATEGORIES = 3
+        CUTOFF = 2
+
+        module_function
+
+        # @param counts [Hash{String => Integer}] word => times the input says it.
+        # @param table [Classifier::Table]
+        # @param stopped [Hash] the stoplist.
+        # @return [Array<String>] up to MAX_CATEGORIES node ids, best first.
+        def best(counts, table, stopped)
+          ranked = score(counts, table).map do |id, (score, evidence)|
+            [id, score, evidence, *tie_breakers(table.nodes[id], counts.keys, stopped)]
+          end
+          ranked = ranked.sort_by { |(_id, score, _evidence, *rest)| [-score.round(6), *rest] }
+          cut(ranked).first(MAX_CATEGORIES).map(&:first)
+        end
+
+        # The first row always stays; later rows only if strong?.
+        def cut(ranked)
+          strongest = ranked.map { |row| row[2] }.max
+          ranked.each_with_index.select { |row, i| i.zero? || strong?(row[2], strongest) }.map(&:first)
+        end
+
+        # Distinct input words and how often each occurs. A plural variant of
+        # a word counts as the same word ("pendant" and "Pendants").
+        def word_counts(tokens)
+          tokens.each_with_object({}) do |token, counts|
+            word = counts.keys.find { |known| Classifier.word_match?(token, known) } || token
+            counts[word] = counts.fetch(word, 0) + 1
+          end
+        end
+
+        # @return [Hash{String => Array(Float, Float)}] node id => [score,
+        #   evidence], for nodes scoring above zero. Each word counts once for
+        #   a node, however many of its keywords match ("light" and "lights"
+        #   are two keywords but one word of the input), so a node cannot win
+        #   by listing a word's variants: KEYWORD_WEIGHT when a `keywords`
+        #   entry matches, PARENT_WEIGHT when only a `parent_keywords` entry
+        #   does, times how often the input says the word (1 + ln count: a
+        #   Shopify tag list repeats "Pendant Lights" once per room, which is
+        #   its best evidence, and the log keeps a long tag list from burying
+        #   a different word). `evidence` is the same sum with each word also
+        #   weighted by how rare it is (ln(1 + nodes / nodes it matches)), so
+        #   "kettle" counts for more than "electric".
+        def score(counts, table)
+          scores = Hash.new { |hash, id| hash[id] = [0.0, 0.0] }
+          counts.each do |word, count|
+            contributions(word, count, table).each { |id, weight, rarity| add(scores[id], weight, rarity) }
+          end
+          scores
+        end
+
+        # @return [Array<Array(String, Float, Float)>] [node id, weight,
+        #   rarity] for every node `word` matches.
+        def contributions(word, count, table)
+          own = matching_ids(word, table.own)
+          parent = matching_ids(word, table.parent) - own
+          return [] if own.empty? && parent.empty?
+
+          tf = 1 + Math.log(count)
+          rarity = Math.log(1 + table.nodes.size.fdiv((own + parent).length))
+          own.map { |id| [id, KEYWORD_WEIGHT * tf, rarity] } + parent.map { |id| [id, PARENT_WEIGHT * tf, rarity] }
+        end
+
+        def add(pair, weight, rarity)
+          pair[0] += weight
+          pair[1] += weight * rarity
+        end
+
+        # Ranking by `score` alone is what classifies a long tag list best,
+        # because rare words are mostly noise there (a room name in a
+        # lighting store's tags). The cut uses `evidence`, which a generic
+        # word cannot fill: an id beyond the first stays only if its evidence
+        # reaches 1/CUTOFF of the strongest.
+        def strong?(evidence, strongest) = (evidence * CUTOFF) - strongest > -1e-9
+
+        # What separates nodes with equal scores, best first: the share of
+        # the node's own name the input covers ("Sofas" before "Sofa
+        # Accessories" for "sofa"), then a name with no stoplisted word in it
+        # ("Household Appliances" before "Household Appliance Accessories"),
+        # then fewer keywords (the smaller node is the more specific one),
+        # then the file's own order.
+        def tie_breakers(node, words, stopped)
+          name = node["name"].to_s.split(" > ").last.to_s.downcase.split(/[^\p{Alpha}]+/)
+                             .select { |word| word.length >= Classifier::MIN_WORD_LENGTH }
+          covered = name.count { |part| words.any? { |word| Classifier.word_match?(word, part) } }
+          generic = name.any? { |part| stopped.key?(part) } ? 1 : 0
+          [-covered.fdiv([name.length, 1].max), generic, Array(node["keywords"]).length, node["order"]]
+        end
+
+        # @return [Array<String>] ids of the nodes in `index` with a keyword
+        #   `word` matches.
+        def matching_ids(word, index)
+          keywords_matching(word).flat_map { |keyword| index.fetch(keyword, []) }.uniq
+        end
+
+        # The keywords Classifier.word_match? accepts `word` for: itself, the
+        # singular it is a plural of ("boots" -> "boot", "watches" ->
+        # "watch"), the plural of it ("boot" -> "boots"), and the "y"/"ies"
+        # swap. Looking a word's few candidate keywords up in a hash, rather
+        # than comparing every word with every keyword, keeps a long
+        # product-tag text (Index::Sources::StorefrontProducts,
+        # docs/plans/local-catalogue.md Phase 2) cheap now that nodes carry
+        # hundreds of keywords.
+        def keywords_matching(word)
+          keywords = [word, "#{word}s", "#{word}es"]
+          keywords << word.delete_suffix("s") if word.end_with?("s")
+          keywords << word.delete_suffix("es") if word.end_with?("es")
+          keywords << "#{word[0..-4]}y" if word.end_with?("ies")
+          keywords << "#{word[0..-2]}ies" if word.end_with?("y")
+          keywords
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/classifier/table.rb

@@ -0,0 +1,63 @@
+require "yaml"
+
+module Portage
+  module Cli
+    module Classifier
+      # The shipped categories.yml and the user's ~/.portage/categories.yml,
+      # merged (a user node replaces a shipped node of the same id), with the
+      # keyword => node ids indexes the scoring looks words up in.
+      Table = Struct.new(:nodes, :own, :parent)
+
+      class Table
+        @cache = {}
+
+        class << self
+          # Built once per pair of files and kept for the process, keyed by
+          # the files' mtime and size, so an edit to ~/.portage/categories.yml
+          # still shows up on the next call; classifying a whole catalogue
+          # would otherwise rebuild the index for every product.
+          def for(known_path, user_path)
+            key = [signature(known_path), signature(user_path)]
+            @cache.clear if @cache.size > 8
+            @cache[key] ||= build(known_path, user_path)
+          end
+
+          # @return [Hash] the YAML mapping at `path`; empty when the file is
+          #   missing, unreadable or not a mapping.
+          def load_yaml(path)
+            return {} unless path && File.readable?(path)
+
+            data = YAML.safe_load_file(path)
+            data.is_a?(Hash) ? data : {}
+          rescue StandardError
+            {}
+          end
+
+          private
+
+          def build(known_path, user_path)
+            ordered = {}
+            load_yaml(known_path).each_with_index { |(id, node), i| ordered[id] = node.merge("order" => i) }
+            load_yaml(user_path).each_with_index do |(id, node), i|
+              ordered[id] = node.merge("order" => ordered.size + i)
+            end
+            new(ordered, index(ordered, "keywords"), index(ordered, "parent_keywords"))
+          end
+
+          def index(nodes, field)
+            result = Hash.new { |hash, key| hash[key] = [] }
+            nodes.each { |id, node| Array(node[field]).each { |keyword| result[keyword] << id } }
+            result.default_proc = nil
+            result
+          end
+
+          def signature(path)
+            return nil unless path && File.readable?(path)
+
+            [path, File.mtime(path).to_r, File.size(path)]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/classifier.rb

@@ -1,4 +1,6 @@
 require "yaml"
+require_relative "classifier/table"
+require_relative "classifier/ranking"
 
 module Portage
   module Cli
@@ -18,10 +20,21 @@ module Portage
     # check matches in both directions ("carpet" contains "pet", "chair"
     # contains "hair", "scarf" contains "car") and was a real source of
     # false positives before this became whole-word.
+    #
+    # `known-stores/categories.yml` is generated by `script/categories`: a level-2
+    # node's `keywords` are its own name plus its descendants', `parent_keywords`
+    # are its parent's words, and `category-stoplist.yml` words are left out.
+    # See Ranking for how a text is scored against it.
     module Classifier
       # `known-stores/categories.yml`, from `lib/portage/cli/classifier.rb`.
       KNOWN_PATH = File.expand_path("../../../known-stores/categories.yml", __dir__).freeze
 
+      # Words that are never evidence of a category (`known-stores/category-stoplist.yml`,
+      # `word: why`): merchandising and url words, and words that name a whole family of
+      # nodes. `script/categories` leaves them out of the keywords; `categories_for`
+      # drops them from its input, so the two sides agree.
+      STOPLIST_PATH = File.expand_path("../../../known-stores/category-stoplist.yml", __dir__).freeze
+
       # The user's own additions/overrides — same id overrides a shipped
       # node's keywords, a new id extends the taxonomy. Absent by default;
       # nothing here is required for the shipped file to work.
@@ -44,21 +57,22 @@ module Portage
       # @param known_path [String] override for KNOWN_PATH — specs redirect
       #   this the same way SearchBackends::Allowlist takes its own `path:`.
       # @param user_path [String] override for PATH.
-      # @return [Array<String>] category ids, most keyword hits first. Ties
-      #   keep the shipped file's own order. Empty when nothing matches.
-      def self.categories_for(text, known_path: KNOWN_PATH, user_path: PATH)
-        words = tokenize(text)
-        return [] if words.empty?
-
-        scored = nodes(known_path, user_path).filter_map { |id, node| rank(id, node, words) }
-        scored.sort_by { |(_id, score, order)| [-score, order] }.map(&:first)
+      # @param stoplist_path [String] override for STOPLIST_PATH.
+      # @return [Array<String>] up to MAX_CATEGORIES category ids, best score
+      #   first (ties: see .tie_breakers). Empty when nothing matches.
+      def self.categories_for(text, known_path: KNOWN_PATH, user_path: PATH, stoplist_path: STOPLIST_PATH)
+        stopped = Table.load_yaml(stoplist_path)
+        counts = Ranking.word_counts(tokenize(text).reject { |word| stopped.key?(word) })
+        return [] if counts.empty?
+
+        Ranking.best(counts, Table.for(known_path, user_path), stopped)
       end
 
       # @return [Array<String>] the taxonomy names for `ids`, in order —
       #   `portage browser import` (Phase 3) shows a domain's guessed
       #   categories by name so the user can judge them before saving.
       def self.names_for(ids, known_path: KNOWN_PATH, user_path: PATH)
-        all = nodes(known_path, user_path)
+        all = Table.for(known_path, user_path).nodes
         Array(ids).filter_map { |id| all.dig(id.to_s, "name") }
       end
 
@@ -88,17 +102,6 @@ module Portage
                                                     .select { |word| word.length >= MIN_WORD_LENGTH }
       end
 
-      # --- Scoring one node against the tokenized input ---
-
-      def self.rank(id, node, words)
-        keywords = Array(node["keywords"])
-        hits = keywords.count { |keyword| words.any? { |word| word_match?(word, keyword) } }
-        return nil unless hits.positive?
-
-        [id, hits, node["order"].to_i]
-      end
-      private_class_method :rank
-
       # Whole-word only, plus the plural forms a keyword list and a real
       # query/title actually differ by: an exact match, one plus a trailing
       # "s" or "es" ("boot"/"boots", "watch"/"watches"), or the "y"/"ies"
@@ -129,30 +132,6 @@ module Portage
           (keyword.end_with?("ies") && word == "#{keyword[0..-4]}y")
       end
       private_class_method :ies_y_match?
-
-      # --- Loading and merging the two files ---
-
-      # Re-read on every call rather than cached process-wide: `find` calls
-      # this once or twice per invocation, not in a hot loop, and a cached
-      # copy would miss an edit to ~/.portage/categories.yml until the next
-      # process start.
-      def self.nodes(known_path, user_path)
-        ordered = {}
-        load_yaml(known_path).each_with_index { |(id, node), i| ordered[id] = node.merge("order" => i) }
-        load_yaml(user_path).each_with_index { |(id, node), i| ordered[id] = node.merge("order" => ordered.size + i) }
-        ordered
-      end
-      private_class_method :nodes
-
-      def self.load_yaml(path)
-        return {} unless path && File.readable?(path)
-
-        data = YAML.safe_load_file(path)
-        data.is_a?(Hash) ? data : {}
-      rescue StandardError
-        {}
-      end
-      private_class_method :load_yaml
     end
   end
 end