portage-cli 0.10.0 → 0.12.0

data/known-stores/category-stoplist.yml

@@ -0,0 +1,40 @@
+# Words Classifier never uses as evidence of a category, on either side: script/categories
+# leaves them out of categories.yml, and Classifier.categories_for drops them from its
+# input. Each entry is `word: why`. Both forms of a word are listed, because the stoplist is
+# matched exactly (the keyword match is what normalizes plurals).
+#
+# Two kinds of word belong here: words that describe how something is sold or filed rather
+# than what it is, and words that label a whole family of nodes ("accessories" names 76 of
+# the 192 level-2 nodes). A specific word that is merely ambiguous does not belong here.
+and: function word; appears in taxonomy names ("Hardware & Tools")
+for: function word; appears in taxonomy names ("Filters for ...")
+the: function word
+with: function word ("Vac with Multi-Purpose Head")
+from: function word
+new: merchandising flag; Light Yard tags products "New Collection"
+sale: merchandising flag; a sale tag says nothing about the product
+gift: merchandising flag; "Gift Set" is a way to sell, not a product type
+gifts: merchandising flag; plural of gift
+set: bundle word; "Gift Set" and "Furniture Set" say how it is packaged
+sets: bundle word; plural of set
+kit: bundle word; "drill kit" is the drill
+kits: bundle word; plural of kit
+collection: storefront and url word; matched "Toll Collection Devices" (4488) for every "New Collection" tag
+collections: url and storefront word; the /collections/ path segment of every Shopify collection url
+products: url word; the /products/ path segment of every Shopify product url
+product: url and page-title word
+category: url word; the /category/ path segment
+shop: page-title word ("Cordless Drills | Hardware | Shop")
+accessories: names 76 of the 192 level-2 nodes, so it cannot tell them apart
+accessory: singular of accessories
+supplies: names 25 level-2 nodes ("Pet Supplies", "Office Supplies"), so it cannot tell them apart
+supply: singular of supplies
+equipment: names 21 level-2 nodes, so it cannot tell them apart
+parts: names 21 level-2 nodes ("Vehicle Parts"), so it cannot tell them apart
+replacement: a spare, not a product type; "Replacement Fade Blade Set"
+general: filler word ("General Office Supplies")
+other: filler word; the catch-all name in many taxonomies
+miscellaneous: catch-all product type; JB Hi-Fi files 237 of its first 1,250 products under "MISCELLANEOUS"
+service: generic product-type word; JB Hi-Fi's "TELCO SERVICES" type matched Food Service (135)
+services: plural of service; JB Hi-Fi files 279 of its first 1,250 products under "TELCO SERVICES"
+hand: craft marker; Light Yard tags 146 of its 164 products "British Hand-Made", and "hand" also names 10 level-2 nodes ("Hand Tools")

data/known-stores/category-synonyms.yml

@@ -0,0 +1,15 @@
+# Words the Google taxonomy has no node for, added to a node's keywords by script/categories.
+# `id:` then `word: why`; the why names the golden case (spec/fixtures/classifier_golden.yml)
+# that needs it, by its text. Add a word here only for a case the golden set proves, never
+# in bulk.
+'594':
+  pendant: pendant light
+  sconce: wall sconce
+  bollard: Bollard light Bollard Lights British Hand-Made Driveway Lights Path Lights
+    Patio Lights Wooden Lights £250-£500
+'604':
+  whitegoods: WHITEGOODS Brand:Beko LimitedStock
+'262':
+  telco: TELCO SERVICES Brand:Samsung InStock
+'187':
+  boots: leather boots

data/lib/portage/cli/browser_import/categorize.rb

@@ -21,7 +21,8 @@ module Portage
         PRODUCT_PATH = %r{/products?/[^/?#]+}
 
         # @param rows [Array<Hash>] Readers rows, all for one domain.
-        # @return [Hash{String => Integer}] category id => weight, top 5.
+        # @return [Hash{String => Integer}] category id => weight, top 5 (equal
+        #   weights in the order first seen).
         def self.domain(rows)
           texts = Hash.new(0)
           rows.each { |row| texts[text_of(row)] += row[:visits] }
@@ -29,7 +30,9 @@ module Portage
           texts.each do |text, visits|
             Classifier.categories_for(text).each { |id| tally[id] += visits } unless text.empty?
           end
-          tally.sort_by { |_id, weight| -weight }.first(TOP_CATEGORIES).to_h
+          # `sort_by` isn't stable, so the position breaks ties: first seen wins.
+          tally.each_with_index.sort_by { |(_id, weight), seen| [-weight, seen] }.first(TOP_CATEGORIES)
+               .to_h { |pair, _seen| pair }
         end
 
         # @param kept [Array<Hash>] Importer's kept entries, rows included.

data/lib/portage/cli/buy.rb

@@ -9,6 +9,7 @@ require_relative "payment_methods"
 require_relative "setting"
 require_relative "decisions"
 require_relative "confidence_check"
+require_relative "confidence_state"
 require_relative "checkout_handoff"
 require_relative "money"
 require_relative "notifier"
@@ -122,14 +123,18 @@ module Portage
       #   hand off the real checkout first checks its total against this
       #   (with `quote_currency:`) and reports `quote_changed` instead if it
       #   is higher, in another currency, or missing. See #finish_checkout.
-      # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength -- all keywords; one per flag, plus
+      # @param quote_store [String, nil] the approved quote's store, and
+      # @param quote_title [String, nil] its title — set by `buy --quote`
+      #   alongside quote_total:. Only the confidence check reads them: they
+      #   go into its `approved_quote` (see #confidence_quote).
+      # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength, Metrics/AbcSize -- all keywords; one per flag, plus
       # injectable collaborators, each assigned to its own ivar
       def initialize(url:, query:, qty: 1, payment_token: nil, yes: false, dry_run: false, product_id: nil,
                      auto_open: nil, notify_webhook: nil, handoff_target: nil, confidence_check: nil,
                      transaction_log: nil, max_price: nil, webmcp_bridge: nil, webmcp_mappings: nil,
                      webmcp_mapping_confirm: nil, autofill: nil, webmcp_autofill_confirm: nil, json: false,
-                     quote_total: nil, quote_currency: nil)
-        # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
+                     quote_total: nil, quote_currency: nil, quote_store: nil, quote_title: nil)
+        # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength, Metrics/AbcSize
         raw = url.to_s.strip
         @uri = URI.parse(raw =~ %r{\Ahttps?://}i ? raw : "https://#{raw}")
         @query = query
@@ -151,6 +156,8 @@ module Portage
         @json = json
         @quote_total = quote_total
         @quote_currency = quote_currency
+        @quote_store = quote_store
+        @quote_title = quote_title
         @webmcp_bridge = webmcp_bridge
         @decisions = {}
       end
@@ -510,7 +517,9 @@ module Portage
       # call it against. Builds a cart, reads it back through the (already
       # cart/checkout-capable, per the gate in #webmcp_flow) session so
       # #reconcile_checkout has a `get_cart`-shaped document to check against
-      # (there's no checkout document either), then calls the preset's
+      # (there's no checkout document either), stops there if the quote cap,
+      # the mismatch check or the opt-in confidence check holds it
+      # (#webmcp_cart_held_report), and only then calls the preset's
       # `handoff_checkout` tool directly on the bridge — after the cart
       # read-back, not before, so any post-mutation "page not ready" gap
       # (see Transport's own retry) has already been waited out by then.
@@ -536,24 +545,84 @@ module Portage
         end
         return webmcp_handoff_dry_run_report(products, product, preset) if @dry_run
 
-        created = session.create_cart(line_items: [{ product_id: line_item_id_of(product), quantity: @qty }],
-                                      context: buyer_context, meta: agent_meta)
-        cart = session.get_cart(cart_id: created["id"], meta: agent_meta)
-        warnings = reconcile_checkout(product, cart)
+        @product = product
+        cart = webmcp_build_cart(session, product)
+        held = webmcp_cart_held_report(products, product, cart)
+        return held if held
 
         result = @webmcp_bridge.execute_tool(preset.handoff_checkout, {})
         autofill = attempt_webmcp_autofill(preset)
-        webmcp_handoff_report("webmcp", products, cart.merge("continue_url" => url_from_handoff(result)), warnings,
+        webmcp_handoff_report("webmcp", products, cart.merge("continue_url" => url_from_handoff(result)), [],
                               autofill: autofill)
       end
 
+      # The same gates #finish_checkout runs before anything leaves this
+      # process, in the same order, all before the hand-off tool sends the
+      # tab to checkout and before anything is autofilled: the quote cap,
+      # then the deterministic mismatch stop, then (only when a decision
+      # backend is enabled) the confidence check. The confidence check only
+      # sees a cart the first two let through, so it can hold this hand-off
+      # but never let through one they stop.
+      # @return [Hash, nil] the report for the first gate that held, or nil.
+      def webmcp_cart_held_report(products, product, cart)
+        warnings = reconcile_checkout(product, cart)
+        return quote_changed_report("webmcp", products, cart, warnings) if quote_exceeded?(cart)
+        return webmcp_cart_mismatch_report(products, cart, warnings) if warnings.any?
+
+        webmcp_low_confidence_report(products, cart)
+      end
+
+      # A hold here points at the store's cart page, like the mismatch stop:
+      # checkout was never opened, so there's nothing at a checkout URL yet.
+      def webmcp_low_confidence_report(products, cart)
+        verdict = decide_confidence(cart, [])
+        return nil if verdict.nil? || verdict[:proceed]
+
+        handoff_report("webmcp", products, cart.merge("continue_url" => webmcp_cart_page_url), [],
+                       outcome: "low_confidence",
+                       message: "#{low_confidence_message(verdict)} Checkout wasn't opened, and nothing was " \
+                                "autofilled.")
+      end
+
+      # Adds the line to the store's cart, then reads the cart back.
+      def webmcp_build_cart(session, product)
+        created = session.create_cart(line_items: [{ product_id: line_item_id_of(product), quantity: @qty }],
+                                      context: buyer_context, meta: agent_meta)
+        session.get_cart(cart_id: created["id"], meta: agent_meta)
+      end
+
+      # Same fail-closed rule as #full_buy (see #decide_escalation): a cart
+      # that doesn't match the request stops here, before the hand-off tool
+      # sends the tab to checkout and before anything is autofilled. This
+      # flow used to only add the mismatch to `warnings` and carry on to
+      # checkout. The cart already exists on the store, so the report points
+      # at the store's cart page for the person to inspect, not at checkout.
+      # Every preset with a `handoff_checkout` tool today is Shopify's, where
+      # that page is /cart. A cart has no checkout status, so the verdict is
+      # the mismatch alone: `decisions.escalation.reason` is "mismatch", as
+      # on #full_buy's own stop.
+      def webmcp_cart_mismatch_report(products, cart, warnings)
+        @decisions[:escalation] = Decisions.escalation(checkout_status: nil, warnings: warnings)
+        handoff_report("webmcp", products, cart.merge("continue_url" => webmcp_cart_page_url), warnings,
+                       outcome: "checkout_mismatch",
+                       message: "Stopped before checkout — the store's cart didn't match the request: " \
+                                "#{warnings.join(' ')} Nothing was bought, and checkout wasn't opened.")
+      end
+
+      def webmcp_cart_page_url
+        URI.join(@uri, "/cart").to_s
+      end
+
       # Unlike #full_buy's dry run (which still creates a UCP checkout, since
       # that's a document the store drops on its own), a dry run here stops
       # before `create_cart`: every step after it changes something outside
       # this process — a real cart on the store, the bridge's own tab
       # navigated to checkout, and (with autofill on) typing into that page.
       # None of that is safe to do on a preview run, so this only reports
-      # what the run would have done. No checkout_id either, so History
+      # what the run would have done. That also means there's no cart to
+      # reconcile, so unlike #dry_run_report this can never carry
+      # `checkout_mismatch: true`; the real run checks the cart and stops
+      # there instead. No checkout_id either, so History
       # records it as a search, not a purchase.
       def webmcp_handoff_dry_run_report(products, product, preset)
         build_report(
@@ -764,6 +833,7 @@ module Portage
                               message: no_match_message)
         end
 
+        @product = product
         checkout = session.create_checkout(line_items: [{ product_id: line_item_id_of(product), quantity: @qty }],
                                            fulfillment: requested_fulfillment(fulfillment_adapter),
                                            context: buyer_context, meta: agent_meta)
@@ -779,34 +849,78 @@ module Portage
       # (if surprising) checkout response, not a transport error. Buying
       # blind against a mismatch this method could have caught defeats the
       # point of an agent shopping on the buyer's behalf, so this always
-      # checks and surfaces what it finds; PORTAGE_ABORT_ON_CHECKOUT_MISMATCH
-      # additionally refuses to proceed rather than merely warning.
+      # checks, and any warning it returns stops a real purchase before
+      # payment (see #decide_escalation). The quoted total and currency are
+      # #quote_exceeded?'s job, not this method's.
+      #
+      # This is the authoritative check, and it always runs first: the
+      # opt-in model check (#decide_confidence) only ever sees a checkout
+      # that already passed it, and can hold that checkout but never
+      # overrule a warning from here.
       def reconcile_checkout(product, checkout)
         item_id = line_item_id_of(product)
-        line = Array(checkout["line_items"]).find { |li| li.dig("item", "id") == item_id }
+        lines = Array(checkout["line_items"])
+        line = lines.find { |li| li.dig("item", "id") == item_id }
         return ["Store dropped the requested item (#{item_id}) from checkout."] unless line
 
         warnings = []
         if line["quantity"] != @qty
           warnings << "Store checked out quantity #{line['quantity']}, not the requested #{@qty}."
         end
+        warnings + price_mismatches(product, item_id, line, checkout["currency"]) +
+          unrequested_lines(lines.reject { |li| li.equal?(line) })
+      end
+
+      # Only one line is ever requested, so any other line is something the
+      # person never asked for: an upsell, a "shipping protection" add-on,
+      # or (on a WebMCP cart) whatever was already sitting in the store's
+      # cart. One that costs nothing is let through, so a store's free gift
+      # or $0 sample doesn't stop the purchase; one whose cost can't be read
+      # is treated as costing something, so it stops.
+      def unrequested_lines(extras)
+        extras.reject { |li| line_cost(li)&.zero? }.map do |li|
+          label = [li.dig("item", "id"), li.dig("item", "title")].compact.join(" ")
+          "Store added #{label.empty? ? 'a line' : label} to checkout, which wasn't requested."
+        end
+      end
+
+      # The line's own total, else its unit price times its quantity; nil
+      # when the line carries neither.
+      def line_cost(line)
+        total = Portage::Ucp::Support::Totals.amount(line["totals"])
+        return total if total.is_a?(Numeric)
+
+        price = line.dig("item", "price")
+        quantity = line["quantity"] || 1
+        price * quantity if price.is_a?(Numeric) && quantity.is_a?(Numeric)
+      end
+
+      # A unit price is only comparable in the catalog's own currency, so a
+      # checkout in another one is reported as that, not as a price change.
+      def price_mismatches(product, item_id, line, currency)
+        catalog_currency = expected_unit_currency(product, item_id)
+        if catalog_currency && currency && catalog_currency != currency
+          return ["Store checked out in #{currency}, not the catalog's #{catalog_currency}."]
+        end
 
         expected = expected_unit_price(product, item_id)
         actual = line.dig("item", "price")
-        if expected && actual && expected != actual
-          warnings << "Store priced the item at #{actual} #{checkout['currency']} minor units per unit, " \
-                      "not the catalog's #{expected}."
-        end
-        warnings
+        return [] unless expected && actual && expected != actual
+
+        ["Store priced the item at #{actual} #{currency} minor units per unit, not the catalog's #{expected}."]
       end
 
       def expected_unit_price(product, item_id)
-        variant = Array(product["variants"]).find { |v| v["id"] == item_id }
-        variant&.dig("price", "amount") || product.dig("price_range", "min", "amount")
+        expected_price_of(product, item_id)&.dig("amount")
       end
 
-      def abort_on_mismatch?
-        Setting.flag?(env: "PORTAGE_ABORT_ON_CHECKOUT_MISMATCH")
+      def expected_unit_currency(product, item_id)
+        expected_price_of(product, item_id)&.dig("currency")
+      end
+
+      def expected_price_of(product, item_id)
+        variant = Array(product["variants"]).find { |v| v["id"] == item_id }
+        variant&.dig("price") || product.dig("price_range", "min")
       end
 
       # Submits PORTAGE_SHIP_* (see Portage::Cli::ShippingProfile) as the
@@ -963,22 +1077,38 @@ module Portage
         escalation = decide_escalation(checkout, warnings)
         return escalated_report(source, products, checkout, warnings, escalation) if escalation[:escalate]
         return dry_run_report(source, products, checkout, warnings) if @dry_run
-        return webmcp_handoff_report(source, products, checkout, warnings) if force_handoff
+        return webmcp_handoff_checkout_report(source, products, checkout, warnings) if force_handoff
         return confirmation_needed_report(source, products, checkout, warnings) unless confirmed?
 
         complete(session, source, products, checkout, warnings)
       end
 
+      # The non-preset WebMCP hand-off (`express_stop` through #full_buy). Same
+      # last gate as #webmcp_handoff_checkout_flow's: with a decision backend
+      # enabled, the confidence check runs before anything is handed off, on a
+      # checkout the quote cap and the mismatch stop already let through. A
+      # hold reports `low_confidence` pointing at the store's cart page, as the
+      # preset flow's does, so nothing is sent to a `profile` target's checkout.
+      # No backend named: straight to the hand-off, as before.
+      def webmcp_handoff_checkout_report(source, products, checkout, warnings)
+        webmcp_low_confidence_report(products, checkout) ||
+          webmcp_handoff_report(source, products, checkout, warnings)
+      end
+
       # First in #finish_checkout, ahead of the escalation gates: a
       # `quote_changed` refusal never hands off, since that would spend the
       # quote and open a checkout the person never approved. #complete is
       # the only place this file charges, and it is reached only through
       # #finish_checkout.
+      #
+      # A quote with no total (its dry run had none to show, as a WebMCP
+      # preset dry run never does) caps nothing, so it refuses too, rather
+      # than buying at whatever the store now asks.
       def quote_exceeded?(checkout)
-        return false unless @quote_total
+        return false unless quote_run?
 
         total = checkout_total(checkout)
-        total.nil? || total > @quote_total || checkout["currency"] != @quote_currency
+        @quote_total.nil? || total.nil? || total > @quote_total || checkout["currency"] != @quote_currency
       end
 
       def quote_changed_report(source, products, checkout, warnings)
@@ -990,6 +1120,11 @@ module Portage
       end
 
       def quote_changed_message(total, currency)
+        if @quote_total.nil?
+          return "The quote has no total to hold this checkout (now #{quoted_amount(total, currency)}) to — " \
+                 "nothing was bought. Dry-run again for a priced quote."
+        end
+
         "The price changed since the quote (was #{quoted_amount(@quote_total, @quote_currency)}, " \
           "now #{quoted_amount(total, currency)}) — nothing was bought."
       end
@@ -998,15 +1133,18 @@ module Portage
 
       # Hand off vs. keep going is Decisions.escalation's call
       # (docs/plans/system-one-decision-layer.md § Responsibilities 2): a
-      # literal `requires_escalation` status always escalates. A mismatch
-      # from #reconcile_checkout escalates only under
-      # PORTAGE_ABORT_ON_CHECKOUT_MISMATCH. By default the warnings are
-      # surfaced on the report, and the purchase is not stopped for them.
+      # literal `requires_escalation` status always escalates, and so does
+      # any mismatch from #reconcile_checkout, with no setting to turn that
+      # off. This used to only warn unless PORTAGE_ABORT_ON_CHECKOUT_MISMATCH
+      # was set, so by default a checkout that didn't match what the person
+      # approved was still paid for and reported `purchased`. That variable
+      # is now ignored. A --dry-run never pays, so there the mismatch stays
+      # in `warnings` and #dry_run_report flags it instead, since an agent
+      # needs the priced checkout to tell the person what went wrong.
       # The verdict lands on the report's `decisions:`, and the report's
       # `outcome:` names which gate (if any) stopped the purchase.
       def decide_escalation(checkout, warnings)
-        verdict = Decisions.escalation(checkout_status: checkout["status"],
-                                       warnings: abort_on_mismatch? ? warnings : [])
+        verdict = Decisions.escalation(checkout_status: checkout["status"], warnings: @dry_run ? [] : warnings)
         @decisions[:escalation] = verdict
         verdict
       end
@@ -1129,17 +1267,48 @@ module Portage
         Portage::Ucp::Support::Totals.amount(checkout["totals"])
       end
 
-      # Never includes the payment token: the state goes to a model backend,
-      # which may be a hosted API (Jev).
+      # Runs only on a checkout that already passed #reconcile_checkout, the
+      # quote cap and the spend policy, so it can hold a purchase but never
+      # let through one those would stop. The state is ConfidenceState's
+      # allowlisted summary — never the payment token, the address or the
+      # buyer's contact details — because it goes to a model backend that
+      # may be a hosted third-party API (Jev).
       def decide_confidence(checkout, warnings)
-        verdict = confidence_check.call(
-          query: @query, merchant: @uri.host, quantity: @qty, warnings: warnings,
-          checkout: checkout.slice("id", "status", "currency", "line_items", "totals")
-        )
+        return nil unless confidence_check.enabled?
+
+        verdict = confidence_check.call(confidence_state(checkout, warnings))
         @decisions[:confidence] = verdict if verdict
         verdict
       end
 
+      def confidence_state(checkout, warnings)
+        item_id = @product && line_item_id_of(@product)
+        ConfidenceState.build(
+          request: { query: @query, merchant: @uri.host, quantity: @qty, item_id: item_id,
+                     item_title: picked_title(item_id) },
+          checkout: checkout, warnings: warnings, quote: confidence_quote
+        )
+      end
+
+      # The picked product's title from the store's own search, plus its
+      # variant's when the checked-out variant has one ("Tee — Large").
+      def picked_title(item_id)
+        return nil unless @product
+
+        [@product["title"], variant_matching(@product, item_id)&.dig("title")].compact.uniq.join(" — ")
+      end
+
+      # What `buy --quote` pinned, as the person approved it. nil on a run
+      # with no quote.
+      def confidence_quote
+        return nil unless quote_run?
+
+        { store: @quote_store, product_id: @product_id, title: @quote_title, quantity: @qty,
+          total: @quote_total, currency: @quote_currency }
+      end
+
+      def quote_run? = !(@quote_store || @quote_total || @quote_currency).nil?
+
       def confidence_check
         @confidence_check ||= ConfidenceCheck.new
       end
@@ -1410,9 +1579,18 @@ module Portage
         @yes
       end
 
+      # `warnings` here are #reconcile_checkout's mismatches, which a real
+      # run of the same checkout stops on — so the report says so up front,
+      # before anyone approves a quote for it.
       def dry_run_report(source, products, checkout, warnings = [])
-        checkout_report(source, products, checkout, outcome: "dry_run", warnings: warnings,
-                                                    message: "Dry run — checkout created but not completed.")
+        message = "Dry run — checkout created but not completed."
+        extra = {}
+        if warnings.any?
+          message += " It doesn't match the request (#{warnings.join(' ')}), so a real purchase would stop " \
+                     "with checkout_mismatch."
+          extra[:checkout_mismatch] = true
+        end
+        checkout_report(source, products, checkout, outcome: "dry_run", warnings: warnings, message: message, **extra)
       end
 
       def confirmation_needed_report(source, products, checkout, warnings = [])

data/lib/portage/cli/check.rb

@@ -57,11 +57,21 @@ module Portage
         report = report.merge(adapter: adapter_for(report))
         report = report.merge(webmcp: webmcp_for(report))
         verdict = verdict_for(report)
-        report.merge(verdict: verdict, next_step: CheckNextStep.call(verdict, report))
+        with_index_hint(report.merge(verdict: verdict, next_step: CheckNextStep.call(verdict, report)))
       end
 
       private
 
+      # docs/plans/local-catalogue.md Phase 2: a Shopify (or native UCP)
+      # store's catalogue can be crawled into the local index. Check only
+      # names the command; it never crawls.
+      def with_index_hint(report)
+        return report unless report[:native_ucp] || report[:platform] == "Shopify"
+
+        port = @uri.port == @uri.default_port ? "" : ":#{@uri.port}"
+        report.merge(index_hint: "portage index add #{@uri.scheme}://#{@uri.host}#{port} --crawl")
+      end
+
       def handoff_only_report
         { url: @uri.to_s, native_ucp: nil, platform: nil, recommended_gem: nil, handoff_only: true, adapter: nil,
           webmcp: webmcp_skipped("#{@uri.host} is hand-off only, so it isn't probed"),

data/lib/portage/cli/classifier/ranking.rb

@@ -0,0 +1,135 @@
+module Portage
+  module Cli
+    module Classifier
+      # Scores the tokenized input against every node and keeps the best few
+      # (docs/plans/local-catalogue.md, Phase 5). Nodes carry hundreds of
+      # rolled-up keywords, so this is where "a generic word matches a dozen
+      # categories" gets dealt with.
+      module Ranking
+        # A node's own name words and its descendants' (`keywords`) say what
+        # the product is; its parent's (`parent_keywords`, e.g. "home",
+        # "garden") only where it is filed, so they count half as much.
+        KEYWORD_WEIGHT = 2
+        PARENT_WEIGHT = 1
+
+        # Callers treat every returned id as evidence (store routing, a
+        # browser domain's category tally), so the answer keeps only the best
+        # MAX_CATEGORIES ids and, of those, only ones scoring at least
+        # 1/CUTOFF of the best: "electric kettle" is Kitchen & Dining, not
+        # also Chairs because of "electric".
+        MAX_CATEGORIES = 3
+        CUTOFF = 2
+
+        module_function
+
+        # @param counts [Hash{String => Integer}] word => times the input says it.
+        # @param table [Classifier::Table]
+        # @param stopped [Hash] the stoplist.
+        # @return [Array<String>] up to MAX_CATEGORIES node ids, best first.
+        def best(counts, table, stopped)
+          ranked = score(counts, table).map do |id, (score, evidence)|
+            [id, score, evidence, *tie_breakers(table.nodes[id], counts.keys, stopped)]
+          end
+          ranked = ranked.sort_by { |(_id, score, _evidence, *rest)| [-score.round(6), *rest] }
+          cut(ranked).first(MAX_CATEGORIES).map(&:first)
+        end
+
+        # The first row always stays; later rows only if strong?.
+        def cut(ranked)
+          strongest = ranked.map { |row| row[2] }.max
+          ranked.each_with_index.select { |row, i| i.zero? || strong?(row[2], strongest) }.map(&:first)
+        end
+
+        # Distinct input words and how often each occurs. A plural variant of
+        # a word counts as the same word ("pendant" and "Pendants").
+        def word_counts(tokens)
+          tokens.each_with_object({}) do |token, counts|
+            word = counts.keys.find { |known| Classifier.word_match?(token, known) } || token
+            counts[word] = counts.fetch(word, 0) + 1
+          end
+        end
+
+        # @return [Hash{String => Array(Float, Float)}] node id => [score,
+        #   evidence], for nodes scoring above zero. Each word counts once for
+        #   a node, however many of its keywords match ("light" and "lights"
+        #   are two keywords but one word of the input), so a node cannot win
+        #   by listing a word's variants: KEYWORD_WEIGHT when a `keywords`
+        #   entry matches, PARENT_WEIGHT when only a `parent_keywords` entry
+        #   does, times how often the input says the word (1 + ln count: a
+        #   Shopify tag list repeats "Pendant Lights" once per room, which is
+        #   its best evidence, and the log keeps a long tag list from burying
+        #   a different word). `evidence` is the same sum with each word also
+        #   weighted by how rare it is (ln(1 + nodes / nodes it matches)), so
+        #   "kettle" counts for more than "electric".
+        def score(counts, table)
+          scores = Hash.new { |hash, id| hash[id] = [0.0, 0.0] }
+          counts.each do |word, count|
+            contributions(word, count, table).each { |id, weight, rarity| add(scores[id], weight, rarity) }
+          end
+          scores
+        end
+
+        # @return [Array<Array(String, Float, Float)>] [node id, weight,
+        #   rarity] for every node `word` matches.
+        def contributions(word, count, table)
+          own = matching_ids(word, table.own)
+          parent = matching_ids(word, table.parent) - own
+          return [] if own.empty? && parent.empty?
+
+          tf = 1 + Math.log(count)
+          rarity = Math.log(1 + table.nodes.size.fdiv((own + parent).length))
+          own.map { |id| [id, KEYWORD_WEIGHT * tf, rarity] } + parent.map { |id| [id, PARENT_WEIGHT * tf, rarity] }
+        end
+
+        def add(pair, weight, rarity)
+          pair[0] += weight
+          pair[1] += weight * rarity
+        end
+
+        # Ranking by `score` alone is what classifies a long tag list best,
+        # because rare words are mostly noise there (a room name in a
+        # lighting store's tags). The cut uses `evidence`, which a generic
+        # word cannot fill: an id beyond the first stays only if its evidence
+        # reaches 1/CUTOFF of the strongest.
+        def strong?(evidence, strongest) = (evidence * CUTOFF) - strongest > -1e-9
+
+        # What separates nodes with equal scores, best first: the share of
+        # the node's own name the input covers ("Sofas" before "Sofa
+        # Accessories" for "sofa"), then a name with no stoplisted word in it
+        # ("Household Appliances" before "Household Appliance Accessories"),
+        # then fewer keywords (the smaller node is the more specific one),
+        # then the file's own order.
+        def tie_breakers(node, words, stopped)
+          name = node["name"].to_s.split(" > ").last.to_s.downcase.split(/[^\p{Alpha}]+/)
+                             .select { |word| word.length >= Classifier::MIN_WORD_LENGTH }
+          covered = name.count { |part| words.any? { |word| Classifier.word_match?(word, part) } }
+          generic = name.any? { |part| stopped.key?(part) } ? 1 : 0
+          [-covered.fdiv([name.length, 1].max), generic, Array(node["keywords"]).length, node["order"]]
+        end
+
+        # @return [Array<String>] ids of the nodes in `index` with a keyword
+        #   `word` matches.
+        def matching_ids(word, index)
+          keywords_matching(word).flat_map { |keyword| index.fetch(keyword, []) }.uniq
+        end
+
+        # The keywords Classifier.word_match? accepts `word` for: itself, the
+        # singular it is a plural of ("boots" -> "boot", "watches" ->
+        # "watch"), the plural of it ("boot" -> "boots"), and the "y"/"ies"
+        # swap. Looking a word's few candidate keywords up in a hash, rather
+        # than comparing every word with every keyword, keeps a long
+        # product-tag text (Index::Sources::StorefrontProducts,
+        # docs/plans/local-catalogue.md Phase 2) cheap now that nodes carry
+        # hundreds of keywords.
+        def keywords_matching(word)
+          keywords = [word, "#{word}s", "#{word}es"]
+          keywords << word.delete_suffix("s") if word.end_with?("s")
+          keywords << word.delete_suffix("es") if word.end_with?("es")
+          keywords << "#{word[0..-4]}y" if word.end_with?("ies")
+          keywords << "#{word[0..-2]}ies" if word.end_with?("y")
+          keywords
+        end
+      end
+    end
+  end
+end