portage-cli 0.7.5 → 0.9.0

data/lib/portage/cli/product_page.rb

@@ -0,0 +1,84 @@
+require "uri"
+require_relative "browser_opener"
+
+module Portage
+  module Cli
+    # Opens the store's page for an offer or quote, so the person can look
+    # before they pick or approve (docs/plans/human-pick-and-approve.md
+    # Phase 2, "View the product page"). Used by `pick --view`, `approve
+    # --view` and the `v`/`v N` answers at a tty prompt. Never gated by the
+    # auto-open setting: the person asked for this page by name.
+    #
+    # The URL came from the store, so it's untrusted: only an http(s) URL on
+    # the offer's own store host is opened, and anything else is refused
+    # with `view_refused` rather than handed to the OS opener.
+    #
+    # Host rule: the URL's host must equal the store's host, compared
+    # case-insensitively with one leading `www.` ignored on either side
+    # (`shop.example` and `www.shop.example` are the same shop). Any other
+    # subdomain (`cdn.shop.example`, `shop.example.evil.test`) is refused —
+    # a store that keeps product pages on another host just can't be
+    # viewed from here. A URL carrying credentials (`user@host`) is refused
+    # too.
+    class ProductPage
+      include BrowserOpener
+
+      # @param url [String, nil] the product page as find/compare returned it.
+      # @param store [String] the offer's store origin (or a bare host).
+      def initialize(url:, store:)
+        @url = url
+        @store = store
+      end
+
+      # @return [Hash] `outcome: "viewed"` (with `opened:`) or
+      #   `outcome: "view_refused"` (with the reason as `message:`).
+      def open
+        reason = refusal
+        return { outcome: "view_refused", url: @url, store: @store, message: reason } if reason
+
+        opened = open_browser(@url)
+        { outcome: "viewed", url: @url, opened: opened,
+          message: opened ? "Opened #{@url}." : "Couldn't open a browser — the page is #{@url}" }
+      end
+
+      # @return [String, nil] why the page can't be opened, nil when it can.
+      def refusal
+        return "No product page on record for this offer." if @url.to_s.strip.empty?
+
+        uri = web_uri(@url)
+        return "Not an http(s) URL: #{@url}" unless uri
+        return "Refusing a URL with credentials in it: #{@url}" if uri.userinfo
+        return nil if self.class.same_shop?(uri.host, store_host)
+
+        "#{uri.host} isn't the offer's store (#{store_host || @store}) — not opening #{@url}"
+      end
+
+      def self.same_shop?(host, store_host)
+        return false unless host && store_host
+
+        host_key(host) == host_key(store_host)
+      end
+
+      def self.host_key(host) = host.downcase.delete_prefix("www.")
+      private_class_method :host_key
+
+      private
+
+      def store_host
+        raw = @store.to_s.strip
+        parse(raw.match?(%r{\Ahttps?://}i) ? raw : "https://#{raw}")&.host
+      end
+
+      def web_uri(url)
+        uri = parse(url)
+        uri if uri&.host && %w[http https].include?(uri.scheme&.downcase)
+      end
+
+      def parse(url)
+        URI.parse(url.to_s.strip)
+      rescue URI::InvalidURIError
+        nil
+      end
+    end
+  end
+end

data/lib/portage/cli/quotes.rb

@@ -0,0 +1,82 @@
+require "json"
+require "fileutils"
+require "securerandom"
+
+module Portage
+  module Cli
+    # Local record of the priced checkouts `portage buy --dry-run` has shown,
+    # one JSON file per quote under ~/.portage/quotes/. A quote is what
+    # `buy --quote QUOTE_ID --yes` later buys: it pins the store, product,
+    # quantity and the total that was shown, so the run that charges can
+    # refuse if the price has moved since (see Cli.buy_from_quote).
+    #
+    # Quotes never expire. Each is single use: #consume stamps `used_at`
+    # rather than deleting the file, so a spent quote is still told apart
+    # from one that never existed.
+    class Quotes
+      DIR = File.join(Dir.home, ".portage", "quotes").freeze
+      ID_FORMAT = /\Aqt_[0-9a-f]{12}\z/
+
+      def initialize(dir: DIR, now: Time.now)
+        @dir = dir
+        @now = now.to_i
+      end
+
+      # @param total [Integer, nil] minor units of `currency`.
+      # @param title [String, nil] what the checkout holds, for `portage
+      #   approve`'s summary.
+      # @param url [String, nil] the offer's product page, for `approve --view`.
+      # @return [Hash, nil] the saved quote (string keys, `quote_id` set), or
+      #   nil when it couldn't be written — a quote that can't be saved just
+      #   isn't offered, never a failed dry run.
+      def create(store:, product_id:, qty:, total:, currency:, query: nil, offer_ref: nil, title: nil, url: nil)
+        quote = { "quote_id" => "qt_#{SecureRandom.hex(6)}", "offer_ref" => offer_ref, "store" => store,
+                  "product_id" => product_id, "query" => query, "qty" => qty, "total" => total,
+                  "currency" => currency, "title" => title, "url" => url, "created_at" => @now,
+                  "approved" => false }
+        write(quote)
+      end
+
+      # @return [Hash, nil] nil for an unknown (or malformed) id.
+      def find(quote_id)
+        return nil unless quote_id.to_s.match?(ID_FORMAT)
+
+        parsed = JSON.parse(File.read(path(quote_id)))
+        parsed if parsed.is_a?(Hash)
+      rescue StandardError
+        nil
+      end
+
+      # Records the person's yes to this quote's total
+      # (docs/plans/human-pick-and-approve.md Phase 2): `by` is "person" for
+      # a yes typed at the tty, "agent_relayed" for one an agent passed on.
+      # A relayed yes never downgrades a quote the person already approved.
+      # @return [Hash, nil] the updated quote, nil when unknown or unwritable.
+      def approve(quote_id, by:)
+        quote = find(quote_id)
+        return nil unless quote
+        return quote if quote["approved_by"] == "person"
+
+        write(quote.merge("approved" => true, "approved_by" => by, "approved_at" => @now))
+      end
+
+      # Marks the quote spent. Best-effort, like every other local record.
+      def consume(quote_id)
+        quote = find(quote_id)
+        write(quote.merge("used_at" => @now)) if quote
+      end
+
+      private
+
+      def path(quote_id) = File.join(@dir, "#{quote_id}.json")
+
+      def write(quote)
+        FileUtils.mkdir_p(@dir)
+        File.write(path(quote["quote_id"]), JSON.generate(quote))
+        quote
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/portage/cli/search_backends.rb

@@ -5,6 +5,10 @@ require "yaml"
 require "portage/ucp"
 require "portage/ucp/support/connection"
 require_relative "user_agent"
+require_relative "classifier"
+require_relative "index/store"
+require_relative "index/product_store"
+require_relative "index/known_cache"
 
 module Portage
   module Cli
@@ -31,12 +35,23 @@ module Portage
       ].freeze
 
       # Ordered cheapest/most-trusted first: your own allowlist costs no
-      # network call and needs no key, DuckDuckGo needs no key, the keyed
-      # engines only participate when their credentials are actually present.
+      # network call and needs no key, the local index next (still no
+      # network call, but untrusted — see Index's own comment), DuckDuckGo
+      # needs no key, the keyed engines only participate when their
+      # credentials are actually present.
       def self.default
-        [Allowlist.new, DuckDuckGo.new, Brave.new, GoogleCse.new].select(&:available?)
+        [Allowlist.new, Index.new, DuckDuckGo.new, Brave.new, GoogleCse.new].select(&:available?)
       end
 
+      # True when the only thing standing between the caller and a real web
+      # search is a missing API key — i.e. DuckDuckGo's entity-only Instant
+      # Answer API is running solo, with no allowlist and no keyed backend
+      # (Brave/Google CSE) to cover the open-ended queries it can't answer.
+      # Used to decide whether "no candidates" is worth a nudge to set
+      # BRAVE_SEARCH_API_KEY / GOOGLE_CSE_KEY+GOOGLE_CSE_CX (see Find and
+      # Doctor) rather than a plain "nothing found".
+      def self.only_duckduckgo?(backends) = backends.map(&:name) == ["duckduckgo"]
+
       def self.get_json(uri, params: {}, headers: {})
         uri = uri.dup
         uri.query = URI.encode_www_form(params) unless params.empty?
@@ -63,14 +78,33 @@ module Portage
       end
 
       # Stores you've already decided you trust, listed in
-      # `~/.portage/stores.yml` (a bare YAML array of URLs) or `PORTAGE_STORES`
-      # (comma-separated — the PATH-style colon can't separate values that
-      # contain `https://`). Query-independent on purpose: the point of the file
-      # is "always consider these", and the store's own catalog search is what
-      # decides whether it stocks the thing.
+      # `~/.portage/stores.yml` (a bare YAML array of URLs, or an array
+      # mixing in `{url:, categories: [...]}` entries once you've tagged
+      # some) or `PORTAGE_STORES` (comma-separated — the PATH-style colon
+      # can't separate values that contain `https://`).
+      #
+      # Trust stays query-independent: every entry is *always* a candidate
+      # to `find`, on the same footing whether it's tagged or not, and the
+      # store's own catalog search is still what decides whether it stocks
+      # the thing. `categories:` only changes which of these already-trusted
+      # entries get spent on *this* query's dozen probe slots — a stores.yml
+      # with fifty tagged stores across a dozen categories used to crowd out
+      # web-search candidates on every single query; #search now routes by
+      # the query's own category instead of just taking the first N.
       class Allowlist
         PATH = File.join(Dir.home, ".portage", "stores.yml").freeze
 
+        # At most this many of a matching category's tagged stores per
+        # `#search` call, so one heavily-tagged category can't fill every
+        # probe slot by itself.
+        PER_CATEGORY_CAP = 3
+
+        # The hard ceiling regardless of the caller's own `limit:` — mirrors
+        # Find::MAX_PROBES (duplicated rather than required, to avoid
+        # search_backends.rb depending on find.rb) since this backend can be
+        # constructed and searched outside Find too.
+        TOTAL_CAP = 12
+
         def initialize(path: PATH, env: ENV.fetch("PORTAGE_STORES", nil))
           @path = path
           @env = env
@@ -80,23 +114,314 @@ module Portage
 
         def available? = !entries.empty?
 
-        def search(_query, limit: 10) = entries.first(limit)
+        # @return [Array<String>] URLs, routed by #categorized_search.
+        def search(query, limit: 10) = categorized_search(query, [limit, TOTAL_CAP].min).map { |e| e[:url] }
+
+        # @return [Array<Hash>] every entry (url:, categories:), untouched
+        #   by any query — Index::Sources::StoresFile reuses this to seed
+        #   the local index from the same file, rather than re-parsing
+        #   stores.yml itself.
+        def stores = entries
 
         private
 
+        # Named entries always come first (an explicit "buy from <store>"
+        # outranks a category guess), then up to PER_CATEGORY_CAP tagged
+        # entries per matching category not already named. When no tagged
+        # entry matches any of the query's categories at all, this falls
+        # back to named entries plus every untagged entry — the pre-Phase-
+        # 2a behaviour for an all-untagged stores.yml — but never to every
+        # entry once a tagged match exists: a tagged-but-unrelated store
+        # stays excluded, which is the crowding this routing exists to fix.
+        def categorized_search(query, limit)
+          named = named_entries(query)
+          category_ids = Classifier.categories_for(query)
+          return fallback(named, limit) unless any_tagged_match?(category_ids)
+
+          remaining = [limit - named.length, 0].max
+          matched = by_category(category_ids, remaining, named)
+          (named + matched).uniq { |e| e[:url] }.first(limit)
+        end
+
+        def any_tagged_match?(category_ids)
+          category_ids.any? { |category_id| entries.any? { |e| e[:categories].include?(category_id) } }
+        end
+
+        # Named entries plus every untagged entry, capped — not every
+        # entry: a tagged store whose categories didn't match the query
+        # stays out, the same as it would once a tagged match exists.
+        def fallback(named, limit)
+          (named + entries.select { |e| e[:categories].empty? }).uniq { |e| e[:url] }.first(limit)
+        end
+
+        def by_category(category_ids, limit, exclude)
+          picked = []
+          category_ids.each do |category_id|
+            entries_for_category(category_id, exclude).each do |entry|
+              break if picked.length >= limit
+
+              picked << entry unless picked.include?(entry)
+            end
+          end
+          picked
+        end
+
+        def entries_for_category(category_id, exclude)
+          entries.select { |e| e[:categories].include?(category_id) && !exclude.include?(e) }.first(PER_CATEGORY_CAP)
+        end
+
+        # host or bare name ("shop" out of "shop.example") mentioned in the
+        # query — the escape hatch for "buy from <store>" regardless of
+        # what it's tagged, or for an untagged store the caller named.
+        def named_entries(query)
+          entries.select { |e| named?(e, query) }
+        end
+
+        def named?(entry, query)
+          host = host_of(entry[:url])
+          return false if host.empty?
+
+          label = host.sub(/\Awww\./, "").split(".").first.to_s
+          downcased = query.to_s.downcase
+          downcased.include?(host) || (label.length > 2 && downcased.include?(label))
+        end
+
+        def host_of(url)
+          URI.parse(url).host.to_s.downcase
+        rescue URI::InvalidURIError
+          ""
+        end
+
         def entries
-          @entries ||= (env_entries + file_entries).map { |e| e.to_s.strip }.reject(&:empty?).uniq
+          @entries ||= (env_entries + file_entries).uniq { |e| e[:url] }
         end
 
-        def env_entries = @env.to_s.split(",")
+        def env_entries
+          @env.to_s.split(",").map(&:strip).reject(&:empty?).map { |url| { url: url, categories: [] } }
+        end
 
         def file_entries
           return [] unless File.readable?(@path)
 
-          Array(YAML.safe_load_file(@path))
+          Array(YAML.safe_load_file(@path)).filter_map { |entry| normalize(entry) }
         rescue StandardError
           []
         end
+
+        # A bare string (the format before Phase 2a, still the common case)
+        # or a `{url:, categories: [...]}` hash — both parse, per the plan's
+        # "the file stays a bare URL list; an entry becomes tagged only when
+        # it carries categories:".
+        def normalize(entry)
+          case entry
+          when String
+            url = entry.strip
+            url.empty? ? nil : { url: url, categories: [] }
+          when Hash
+            normalize_hash(entry)
+          end
+        end
+
+        def normalize_hash(entry)
+          entry = entry.transform_keys(&:to_s)
+          url = entry["url"].to_s.strip
+          return nil if url.empty?
+
+          { url: url, categories: Array(entry["categories"]).map(&:to_s) }
+        end
+      end
+
+      # `~/.portage/index/stores.json` — origins `portage index build`
+      # found and verified itself (docs/plans/buy-skill-and-local-browser.md
+      # Phase 2b). Unlike Allowlist, this data is **untrusted**: nothing
+      # here was ever typed in by the user, so an entry never skips a probe
+      # (Find still re-verifies it through ProbeCache like any other
+      # candidate URL) and never becomes a `merchant_allowlist`/`--yes`
+      # shortcut — it's just another URL a search backend handed back,
+      # ranked below Allowlist and above the web-search backends in
+      # SearchBackends.default (search_backends_spec.rb has specs proving
+      # both non-shortcuts).
+      #
+      # Routes the same way Allowlist routes a tagged stores.yml (up to
+      # PER_CATEGORY_CAP per matching category, TOTAL_CAP overall), plus
+      # one thing stores.yml can't do: match a query against a *product* the
+      # index has seen (by name or GTIN) and put that product's own stores
+      # first, ahead of a category guess. A store the query names outright
+      # comes next (Phase 3 — the only route an uncategorised
+      # `portage browser import` domain ever gets).
+      #
+      # Phase 2c: also draws on the repo's own known-stores cache
+      # (Index::KnownCache) — fetched lazily the first time this backend is
+      # asked for anything and no cache exists yet — merged *underneath*
+      # the user's own Store/ProductStore entries: a known entry only shows
+      # up when the user's own index doesn't already have that origin/key,
+      # so a local `index add`/`index build` finding always wins. Still the
+      # same untrusted posture either way — an offer built from either
+      # source carries `source: "index"`, never a merchant_allowlist/--yes
+      # shortcut (see this class's own header comment).
+      class Index
+        PER_CATEGORY_CAP = 3
+        TOTAL_CAP = 12
+
+        def initialize(stores: Portage::Cli::Index::Store.new, products: Portage::Cli::Index::ProductStore.new,
+                       known: Portage::Cli::Index::KnownCache.new)
+          @stores = stores
+          @products = products
+          @known = known
+        end
+
+        def name = "index"
+
+        def available? = !store_entries.empty? || !product_entries.empty?
+
+        def search(query, limit: 10)
+          return [] if store_entries.empty? && product_entries.empty?
+
+          product_origins = origins_for_products(query)
+          named = named_origins(query, exclude: product_origins)
+          category_origins = origins_for_categories(Classifier.categories_for(query), exclude: product_origins + named)
+          (product_origins + named + category_origins).uniq.first([limit, TOTAL_CAP].min)
+        end
+
+        private
+
+        def store_entries
+          @store_entries ||= merge_under_own(@stores.all, known_stores, key: "origin")
+        end
+
+        def product_entries
+          @product_entries ||= merge_under_own(@products.all, known_products, key: "key")
+        end
+
+        # `entry[key]` is always present on both sides — Store#upsert seeds
+        # "origin" and ProductStore#upsert seeds "key" the same way on a
+        # brand-new entry, and KnownCache's own fetch just carries whatever
+        # `portage index build --export` wrote, in the same schema.
+        def merge_under_own(own, known, key:)
+          own_keys = own.map { |e| e[key] }
+          own + known.reject { |e| own_keys.include?(e[key]) }
+        end
+
+        def known_stores
+          ensure_known_cache!
+          @known.stores.values
+        end
+
+        def known_products
+          ensure_known_cache!
+          @known.products.values
+        end
+
+        # Fetched at most once per instance — a cache miss found here means
+        # "still no cache", not "try again this call".
+        def ensure_known_cache!
+          return if @known_cache_checked
+
+          @known_cache_checked = true
+          @known.fetch_if_missing!
+        end
+
+        def origins_for_products(query)
+          matches = product_entries.select { |product| product_matches?(product, query) }
+          matches.flat_map { |product| Array(product["stores"]).map { |s| s["origin"] } }.uniq
+        end
+
+        # Whole-word matching, built on the same Classifier.tokenize/
+        # .word_match? a title/query is classified into categories with
+        # (docs/plans/buy-skill-and-local-browser.md Phase 2a) — a plain
+        # substring check goes both ways regardless of word boundaries
+        # ("tea" inside "steam"/"teak", "bag" inside "bagel"), which is
+        # exactly the false-positive class 2a already fixed for category
+        # keywords and this backend was still exposed to.
+        #
+        # A match is either every one of the query's tokens found among the
+        # title/alias's own tokens (a short query naming a longer title,
+        # e.g. "hiking boots" -> "Men's Hiking Boot"), or the reverse (a
+        # longer query that names the whole title/alias as a phrase, e.g.
+        # "where can I buy a trail boot"). An empty/whitespace-only query
+        # tokenizes to nothing and matches no product.
+        def product_matches?(product, query)
+          return true if gtin_match?(product, query)
+
+          # Checked after GTIN, not before: a purely numeric query (the
+          # normal shape of a GTIN) tokenizes to nothing at all —
+          # Classifier.tokenize splits on runs of non-alpha characters — so
+          # gating on "any tokens" first would refuse a valid barcode
+          # lookup before #gtin_match? ever got to compare it.
+          query_tokens = Classifier.tokenize(query)
+          return false if query_tokens.empty?
+
+          [product["title"], *Array(product["aliases"])].compact.any? { |text| title_matches?(query_tokens, text) }
+        end
+
+        # Exact match against the whole (stripped, downcased) query, never a
+        # substring — a GTIN is a barcode, not a word Classifier.tokenize
+        # would even keep (it splits on runs of non-alpha characters, so a
+        # purely numeric query tokenizes to nothing).
+        def gtin_match?(product, query)
+          gtin = product["gtin"]
+          !gtin.to_s.empty? && query.to_s.strip.downcase == gtin.to_s.downcase
+        end
+
+        def title_matches?(query_tokens, text)
+          title_tokens = Classifier.tokenize(text)
+          return false if title_tokens.empty?
+
+          all_match?(query_tokens, title_tokens) || all_match?(title_tokens, query_tokens)
+        end
+
+        def all_match?(these, those)
+          these.all? { |a| those.any? { |b| Classifier.word_match?(a, b) } }
+        end
+
+        # A store the query names outright — its host ("allbirds.com") or
+        # its bare name as a whole word ("buy from allbirds") — whether or
+        # not it carries categories. This is the only way an uncategorised
+        # entry (a `portage browser import` domain whose titles matched no
+        # category — Phase 3) is ever routed: by name, never for a generic
+        # query.
+        def named_origins(query, exclude:)
+          tokens = Classifier.tokenize(query)
+          text = query.to_s.downcase
+          store_entries.filter_map do |entry|
+            origin = entry["origin"]
+            origin if !exclude.include?(origin) && names_store?(origin, text, tokens)
+          end
+        end
+
+        def names_store?(origin, text, tokens)
+          host = URI.parse(origin.to_s).host.to_s.downcase.delete_prefix("www.")
+          return false if host.empty?
+
+          text.include?(host) || tokens.include?(host.split(".").first)
+        rescue URI::InvalidURIError
+          false
+        end
+
+        def origins_for_categories(category_ids, exclude:)
+          picked = []
+          category_ids.each do |category_id|
+            entries_for_category(category_id, exclude + picked).each do |origin|
+              break if picked.length >= TOTAL_CAP
+
+              picked << origin
+            end
+          end
+          picked
+        end
+
+        # Ranked by this category's own weight (how many of the store's
+        # products the Classifier put there — see Index::Builder's
+        # #merge_categories), highest first, before PER_CATEGORY_CAP cuts
+        # it off, so a store barely tagged into a category doesn't take a
+        # slot from one the Classifier weighted heavily into it.
+        def entries_for_category(category_id, exclude)
+          matches = store_entries.select do |e|
+            Array(e["categories"]&.keys).include?(category_id) && !exclude.include?(e["origin"])
+          end
+          ranked = matches.sort_by { |e| -e["categories"][category_id].to_i }
+          ranked.first(PER_CATEGORY_CAP).map { |e| e["origin"] }
+        end
       end
 
       # DuckDuckGo's Instant Answer API — official, documented, no key

data/lib/portage/cli/setup_wizard/prompt.rb

@@ -0,0 +1,67 @@
+require "io/console"
+
+module Portage
+  module Cli
+    class SetupWizard
+      # The one place every step reads a question and prints a line — so
+      # "Enter/blank keeps the current value", "y/N confirm" and "never
+      # echo a secret" (docs/plans/buy-skill-and-local-browser.md Phase 4)
+      # are each implemented once instead of per step. Defaults to the real
+      # $stdin/$stdout, like WebmcpMappingConfirm/BrowserImport::Confirm —
+      # a spec swaps in a fake input/output (or stubs the real $stdin the
+      # same way cli_spec.rb already does for `browser import`).
+      class Prompt
+        def initialize(input: $stdin, output: $stdout)
+          @input = input
+          @output = output
+        end
+
+        def say(text) = @output.puts(text)
+
+        # @return [Boolean] `default` when the answer is blank; otherwise
+        #   whether the answer starts with "y".
+        def confirm(question, default: true)
+          @output.print("#{question} [#{default ? 'Y/n' : 'y/N'}] ")
+          @output.flush
+          answer = @input.gets.to_s.strip.downcase
+          return default if answer.empty?
+
+          answer.start_with?("y")
+        end
+
+        # @return [String, nil] nil for a blank answer — Enter keeps
+        #   whatever's already set, and the step decides what that means.
+        def ask(question, hint: nil)
+          @output.print("#{question}#{" (#{hint})" if hint}: ")
+          @output.flush
+          blank_to_nil(@input.gets.to_s.chomp)
+        end
+
+        # Same contract as #ask, but the answer is never echoed to the
+        # terminal: IO#noecho when `@input` is a real, attached tty (a
+        # spec's stubbed-but-otherwise-real $stdin isn't, so this falls
+        # back to a plain read there, same as a piped stdin in production).
+        def ask_secret(question, hint: nil)
+          @output.print("#{question}#{" (#{hint})" if hint}: ")
+          @output.flush
+          answer = read_secret
+          @output.puts
+          blank_to_nil(answer.to_s.chomp)
+        end
+
+        private
+
+        def blank_to_nil(text)
+          text = text.strip
+          text.empty? ? nil : text
+        end
+
+        def read_secret
+          @input.noecho(&:gets).to_s
+        rescue Errno::ENOTTY, NoMethodError, IOError
+          @input.gets.to_s
+        end
+      end
+    end
+  end
+end