portage-cli 0.7.4 → 0.8.0

data/lib/portage/cli/classifier.rb

@@ -0,0 +1,158 @@
+require "yaml"
+
+module Portage
+  module Cli
+    # "What category is this?" for anything shopping-shaped: a search query,
+    # a catalog product's title/description, a stores.yml/index entry, or
+    # (Phase 3) a browser history/bookmark entry — one classifier, so a
+    # store tagged from its product mix and a query typed by a shopper land
+    # in the same category space and can be matched against each other.
+    #
+    # Plain keyword matching against Google's own published product taxonomy
+    # (https://www.google.com/basepages/producttype/taxonomy-with-ids.en-US.txt),
+    # top two levels only (~200 nodes, `known-stores/categories.yml`, shipped
+    # in the gem so this works offline on a fresh brew/gem install). No LLM
+    # and no network call: it has to run on every `find` and stay instant.
+    # "Plain keyword" means a whole-word match (plus simple plural
+    # normalization, #word_match?), not a substring regex — a substring
+    # check matches in both directions ("carpet" contains "pet", "chair"
+    # contains "hair", "scarf" contains "car") and was a real source of
+    # false positives before this became whole-word.
+    module Classifier
+      # `known-stores/categories.yml`, from `lib/portage/cli/classifier.rb`.
+      KNOWN_PATH = File.expand_path("../../../known-stores/categories.yml", __dir__).freeze
+
+      # The user's own additions/overrides — same id overrides a shipped
+      # node's keywords, a new id extends the taxonomy. Absent by default;
+      # nothing here is required for the shipped file to work.
+      PATH = File.join(Dir.home, ".portage", "categories.yml").freeze
+
+      # `/products/<slug>`, `/collections/<slug>`, `/c/<slug>`,
+      # `/category/<slug>` — the URL shapes stores.yml/index/browser-import
+      # entries actually carry (docs/plans/buy-skill-and-local-browser.md
+      # Phase 2a/3).
+      SLUG_PATTERNS = [
+        %r{/products/([^/?#]+)},
+        %r{/collections/([^/?#]+)},
+        %r{/c/([^/?#]+)},
+        %r{/category/([^/?#]+)}
+      ].freeze
+
+      # @param text [String] a query, a product title/description, or a
+      #   store/product URL. Whichever it is, it's tokenized the same way
+      #   (see #tokenize) so the same node keywords match all of them.
+      # @param known_path [String] override for KNOWN_PATH — specs redirect
+      #   this the same way SearchBackends::Allowlist takes its own `path:`.
+      # @param user_path [String] override for PATH.
+      # @return [Array<String>] category ids, most keyword hits first. Ties
+      #   keep the shipped file's own order. Empty when nothing matches.
+      def self.categories_for(text, known_path: KNOWN_PATH, user_path: PATH)
+        words = tokenize(text)
+        return [] if words.empty?
+
+        scored = nodes(known_path, user_path).filter_map { |id, node| rank(id, node, words) }
+        scored.sort_by { |(_id, score, order)| [-score, order] }.map(&:first)
+      end
+
+      # @return [Array<String>] the taxonomy names for `ids`, in order —
+      #   `portage browser import` (Phase 3) shows a domain's guessed
+      #   categories by name so the user can judge them before saving.
+      def self.names_for(ids, known_path: KNOWN_PATH, user_path: PATH)
+        all = nodes(known_path, user_path)
+        Array(ids).filter_map { |id| all.dig(id.to_s, "name") }
+      end
+
+      # --- Tokenizing the input ---
+
+      # A URL slug is words joined by `-`/`_` (`hand-cut-glass` — see
+      # AGENTS.md's own product terminology); everything else is just
+      # whitespace/punctuation-split. Both a URL's slug words and the plain
+      # words of a title/description/query are kept, so "hiking boots" and
+      # "https://shop.example/products/hiking-boots-mens" tokenize the same
+      # way.
+      # Below 3 letters, a token is noise ("a", "of", "is") rather than a
+      # keyword candidate — every generated keyword is at least this long
+      # too (see the script that built known-stores/categories.yml), so
+      # nothing below this length could ever whole-word match one anyway.
+      MIN_WORD_LENGTH = 3
+
+      # Public rather than private — SearchBackends::Index (Phase 2b) tokenizes
+      # a query and a product's title/aliases the same way this module does
+      # internally, so a query and a product name land in the same word
+      # space instead of re-implementing this split.
+      def self.tokenize(text)
+        string = text.to_s
+        slug_words = SLUG_PATTERNS.filter_map { |pattern| pattern.match(string)&.[](1) }
+                                  .flat_map { |slug| slug.split(/[-_]/) }
+        (slug_words + string.split(/[^\p{Alpha}]+/)).map(&:downcase)
+                                                    .select { |word| word.length >= MIN_WORD_LENGTH }
+      end
+
+      # --- Scoring one node against the tokenized input ---
+
+      def self.rank(id, node, words)
+        keywords = Array(node["keywords"])
+        hits = keywords.count { |keyword| words.any? { |word| word_match?(word, keyword) } }
+        return nil unless hits.positive?
+
+        [id, hits, node["order"].to_i]
+      end
+      private_class_method :rank
+
+      # Whole-word only, plus the plural forms a keyword list and a real
+      # query/title actually differ by: an exact match, one plus a trailing
+      # "s" or "es" ("boot"/"boots", "watch"/"watches"), or the "y"/"ies"
+      # swap ("battery"/"batteries"). Deliberately not a substring check —
+      # substrings match in both directions regardless of word boundaries
+      # ("carpet" contains "pet", "chair" contains "hair", "scarf" contains
+      # "car"), which is real noise, not stemming.
+      # Public — see .tokenize's own comment; SearchBackends::Index (Phase
+      # 2b) matches a query's tokens against a product's title/alias tokens
+      # with this same whole-word (plus plural) rule, rather than a
+      # substring check that goes both ways ("tea" in "steam"/"teak", "bag"
+      # in "bagel").
+      def self.word_match?(word, keyword)
+        word == keyword || plural_of?(word, keyword) || plural_of?(keyword, word) || ies_y_match?(word, keyword)
+      end
+
+      # @return [Boolean] true when `plural` is `singular` plus a trailing
+      # "s" or "es" ("boot"/"boots", "watch"/"watches").
+      def self.plural_of?(plural, singular)
+        ["#{singular}s", "#{singular}es"].include?(plural)
+      end
+      private_class_method :plural_of?
+
+      # @return [Boolean] true when either side is the other's "ies" plural
+      # ("battery"/"batteries").
+      def self.ies_y_match?(word, keyword)
+        (word.end_with?("ies") && keyword == "#{word[0..-4]}y") ||
+          (keyword.end_with?("ies") && word == "#{keyword[0..-4]}y")
+      end
+      private_class_method :ies_y_match?
+
+      # --- Loading and merging the two files ---
+
+      # Re-read on every call rather than cached process-wide: `find` calls
+      # this once or twice per invocation, not in a hot loop, and a cached
+      # copy would miss an edit to ~/.portage/categories.yml until the next
+      # process start.
+      def self.nodes(known_path, user_path)
+        ordered = {}
+        load_yaml(known_path).each_with_index { |(id, node), i| ordered[id] = node.merge("order" => i) }
+        load_yaml(user_path).each_with_index { |(id, node), i| ordered[id] = node.merge("order" => ordered.size + i) }
+        ordered
+      end
+      private_class_method :nodes
+
+      def self.load_yaml(path)
+        return {} unless path && File.readable?(path)
+
+        data = YAML.safe_load_file(path)
+        data.is_a?(Hash) ? data : {}
+      rescue StandardError
+        {}
+      end
+      private_class_method :load_yaml
+    end
+  end
+end

data/lib/portage/cli/compare.rb

@@ -1,4 +1,5 @@
 require_relative "find"
+require_relative "handoff_only"
 
 module Portage
   module Cli
@@ -53,6 +54,8 @@ module Portage
       def resolve_origin
         uri = parse_http(@origin_url)
         return report(message: "Not a store URL: #{@origin_url.inspect}") unless uri
+        return report(message: "#{uri.host} is hand-off only — #{HandoffOnly::LEGAL_NOTICE}") \
+          if @handoff_only.host?(uri.host)
 
         session = discover(origin_of(uri))
         return report(message: "#{origin_of(uri)} doesn't speak UCP.") unless session

data/lib/portage/cli/doctor.rb

@@ -4,6 +4,13 @@ require_relative "confidence_check"
 require_relative "user_agent"
 require_relative "proxy_settings"
 require_relative "shipping_profile"
+require_relative "dot_env"
+require_relative "search_backends"
+require_relative "agent_profile_url"
+require_relative "index"
+require_relative "handoff_target"
+require_relative "handoff_only"
+require_relative "offer_sources"
 
 module Portage
   module Cli
@@ -26,20 +33,50 @@ module Portage
         def to_h = super.compact
       end
 
-      def initialize(adapter_class: nil, proxy_settings: ProxySettings.new, install_doctor: InstallDoctor.new)
+      SELLER_CHECKS = "authenticator, rate limiter, signing keys, payment handlers".freeze
+
+      # @param seller [Boolean] run the seller-side checks against
+      #   Portage::Ucp.configuration. `portage doctor` passes true only when
+      #   --require or --adapter loaded a seller's setup: in a bare shopper
+      #   process that configuration is always the unconfigured default, so
+      #   those four warnings were noise on every fresh install.
+      def initialize(adapter_class: nil, proxy_settings: ProxySettings.new, install_doctor: InstallDoctor.new,
+                     seller: true, dot_env_path: DotEnv.loaded_path, index_stores: Index::Store.new,
+                     index_products: Index::ProductStore.new, known_cache: Index::KnownCache.new)
         @adapter_class = adapter_class
         @proxy_settings = proxy_settings
         @install_doctor = install_doctor
+        @seller = seller
+        @dot_env_path = dot_env_path
+        @index_stores = index_stores
+        @index_products = index_products
+        @known_cache = known_cache
+      end
+
+      # `portage setup` always offers its wizard on a TTY; a bare `portage
+      # doctor`/`configure` only offers it once, on a fresh install, rather
+      # than on every run of someone who's already configured most of this
+      # (docs/plans/buy-skill-and-local-browser.md Phase 4). Conservative on
+      # purpose: true only when *all three* of the foundational settings —
+      # no `.env` file was loaded, no shipping address, no policy at all —
+      # are still untouched. Running `portage index build`/`browser import`
+      # without ever touching shipping/policy still counts as "nothing
+      # configured" here; those are the wizard's own later, opt-in steps,
+      # not what decides whether it's worth offering in the first place.
+      def nothing_configured?
+        @dot_env_path.nil? && ShippingProfile.from_env.nil? && Portage::Ucp::Policy.load.to_h.empty?
       end
 
       def call
         [
           *@install_doctor.findings,
-          authenticator_finding,
-          rate_limiter_finding,
-          signing_keys_finding,
-          payment_handlers_finding,
+          dot_env_finding,
+          *seller_findings,
           decision_backend_finding,
+          search_backend_finding,
+          agent_profile_finding,
+          index_finding,
+          handoff_finding, retailer_offer_sources_finding,
           user_agent_finding,
           shipping_finding,
           proxy_finding,
@@ -52,6 +89,31 @@ module Portage
 
       def config = Portage::Ucp.configuration
 
+      def seller_findings
+        return [authenticator_finding, rate_limiter_finding, signing_keys_finding, payment_handlers_finding] if @seller
+
+        [Finding.new(check: "seller", level: "info",
+                     message: "Seller checks (#{SELLER_CHECKS}) skipped — pass --require with your app's " \
+                              "initializer, or --adapter, to run them.")]
+      end
+
+      # Which env file DotEnv loaded, and a warning when it's readable by
+      # other users: it's where store credentials and payment tokens end up.
+      def dot_env_finding
+        return unless @dot_env_path
+
+        mode = File.stat(@dot_env_path).mode & 0o777
+        details = { path: @dot_env_path, mode: format("%o", mode) }
+        return Finding.new(check: "env_file", level: "info", message: "Loaded #{@dot_env_path}", details: details) \
+          if mode.nobits?(0o077)
+
+        Finding.new(check: "env_file", details: details,
+                    message: "#{@dot_env_path} is readable by other users (mode #{details[:mode]}) and may hold " \
+                             "credentials — run `chmod 600 #{@dot_env_path}`.")
+      rescue SystemCallError
+        nil
+      end
+
       def authenticator_finding
         return unless config.authenticator.is_a?(Portage::Ucp::UnconfiguredAuthenticator)
 
@@ -89,6 +151,132 @@ module Portage
         Finding.new(check: "decision_backend", message: e.message)
       end
 
+      # `portage buy`/`portage find` with no URL/--store depend on a search
+      # backend to name candidate stores in the first place. Every install
+      # gets DuckDuckGo's free Instant Answer API for free, but it only
+      # resolves *entity* queries ("burton snowboards" → burton.com) — an
+      # open-ended query ("coffee", "iphone", "screens") comes back empty
+      # every time, regardless of what any store actually stocks (see
+      # SearchBackends::DuckDuckGo). That's easy to mistake for "find is
+      # broken" rather than "no real web search is configured", so this
+      # flags it up front rather than letting every URL-less search rediscover
+      # it one confusing empty result at a time.
+      def search_backend_finding
+        return if SearchBackends::Allowlist.new.available?
+
+        backends = SearchBackends.default
+        return unless SearchBackends.only_duckduckgo?(backends)
+
+        Finding.new(check: "search_backend", level: "info",
+                    message: "Only DuckDuckGo's free Instant Answer API is active for `portage find`/" \
+                             "`portage buy` without a URL — it resolves specific brand/product names, not " \
+                             "open-ended search terms. Set BRAVE_SEARCH_API_KEY or GOOGLE_CSE_KEY+" \
+                             "GOOGLE_CSE_CX for real web search, or list known stores in " \
+                             "~/.portage/stores.yml.")
+      end
+
+      # `find`/`buy`/OfferSources::ShopifyCatalog need
+      # `meta.ucp-agent.profile` on every catalog/cart/checkout call
+      # (Transports::Http#wire_meta) — PORTAGE_AGENT_PROFILE names it, and
+      # AgentProfileUrl falls back to the repo's own published profile
+      # (docs/agent-profile.md) so a fresh install without one set still
+      # works well enough to try. Always info, never a warning — the
+      # fallback isn't broken, just worth knowing about before a real
+      # deployment.
+      def agent_profile_finding
+        url = AgentProfileUrl.resolve
+        return Finding.new(check: "agent_profile", level: "info", message: "Agent profile: #{url}") \
+          unless ENV.fetch("PORTAGE_AGENT_PROFILE", nil).to_s.strip.empty?
+
+        Finding.new(check: "agent_profile", level: "info",
+                    message: "PORTAGE_AGENT_PROFILE not set — falling back to the repo's own published " \
+                             "profile (#{url}). Run `portage generate agent-profile` and set " \
+                             "PORTAGE_AGENT_PROFILE to your own for production use.")
+      end
+
+      # `portage index build/refresh` (Phase 2b) writes ~/.portage/index/
+      # {stores,products}.json — always info, never a warning: a fresh
+      # install with no index still works, `find` just has less to route
+      # by than one that's run `portage index build`.
+      #
+      # Also one of Phase 2c's three refresh triggers: this refreshes the
+      # known-stores cache (Index::KnownCache) whenever it's stale (more
+      # than 7 days old, which a missing cache counts as too), so a
+      # long-running install's cache doesn't just sit there getting older
+      # every time `find` happens not to trigger the lazy first-run fetch.
+      # One short-timeout GET per file, swallowed on any failure exactly
+      # like KnownCache's own #refresh! — this never turns doctor into a
+      # warning, just a report of whatever the fetch (or its absence)
+      # leaves behind.
+      def index_finding
+        refresh_known_cache_if_stale
+        return Finding.new(check: "index", level: "info", message: "#{index_message}\n#{known_cache_message}") \
+          if @index_stores.exists?
+
+        Finding.new(check: "index", level: "info",
+                    message: "No local index yet — run `portage index build` to give `find` a list of " \
+                             "stores/products on top of stores.yml and web search.\n#{known_cache_message}")
+      end
+
+      def refresh_known_cache_if_stale
+        @known_cache.refresh! if @known_cache.stale?
+      rescue StandardError
+        nil
+      end
+
+      # Phase 7: which of the five retailer OfferSources are on. Always
+      # info — every one of them is opt-in by design, and `find` behaves
+      # identically with none configured. Every one of these still ends in
+      # hand-off no matter how many are enabled, so this finding says that
+      # plainly rather than implying more keys means more automation.
+      def retailer_offer_sources_finding
+        on = OfferSources.retailers.select(&:available?).map(&:name)
+        message = if on.empty?
+                    no_retailer_offer_sources_message
+                  else
+                    "Retailer offer sources active: #{on.join(', ')}. Every one still ends in hand-off; " \
+                      "none can complete a purchase."
+                  end
+        Finding.new(check: "retailer_offer_sources", level: "info", details: { active: on }, message: message)
+      end
+
+      def no_retailer_offer_sources_message
+        "No retailer offer sources configured (Walmart, eBay, Best Buy, Etsy, Amazon) — run `portage " \
+          "setup` to add keys. Every one still ends in hand-off; none can complete a purchase."
+      end
+
+      def index_message
+        age = @index_stores.oldest_verified_age
+        staleness = age ? "oldest entry verified #{age / 86_400} day(s) ago" : "no entries verified yet"
+        "Local index: #{@index_stores.all.length} store(s), #{@index_products.all.length} product(s) " \
+          "(#{staleness}). Run `portage index refresh` to re-verify entries older than 7 days."
+      end
+
+      def known_cache_message
+        unless @known_cache.exists?
+          return "Known-stores list (fetched from the repo, docs/plans/buy-skill-and-local-browser.md " \
+                 "Phase 2c): not fetched yet — offline, or the first fetch hasn't run."
+        end
+
+        age = @known_cache.age
+        days = age ? age / 86_400 : "?"
+        "Known-stores list: #{@known_cache.stores.length} store(s), #{@known_cache.products.length} " \
+          "product(s), fetched #{days} day(s) ago."
+      end
+
+      # docs/plans/buy-skill-and-local-browser.md Phase 5 — always info:
+      # both the current hand-off target and the hand-off-only host list are
+      # legitimate to leave at their defaults. The disclaimer runs here as
+      # well as on every `handoff_only` buy report and the `buy` skill's own
+      # reference doc (decision 5).
+      def handoff_finding
+        target = HandoffTarget.new
+        hosts = HandoffOnly.new.hosts
+        Finding.new(check: "handoff", level: "info", details: { target: target.label, handoff_only_hosts: hosts },
+                    message: "Hand-off target: #{target.label}. Hand-off-only hosts (#{hosts.length}): " \
+                             "#{hosts.join(', ')}. #{HandoffOnly::LEGAL_NOTICE}")
+      end
+
       # PORTAGE_USER_AGENT / config.json's "user_agent" (UserAgent) is sent
       # as-is on every outbound request. Net::HTTP raises ArgumentError on a
       # header value containing CR/LF rather than send it, so a bad override
@@ -115,8 +303,8 @@ module Portage
                                            .select { |var| ENV[var].to_s.empty? }
         return if missing.empty?
 
-        Finding.new(check: "shipping", message: "#{shipping_gap(missing)} Export them in your shell; " \
-                                                ".env.example lists every PORTAGE_SHIP_* variable.")
+        Finding.new(check: "shipping", message: "#{shipping_gap(missing)} Set them in ~/.portage/.env " \
+                                                "(.env.example lists every variable) or your shell.")
       end
 
       def shipping_gap(missing)

data/lib/portage/cli/dot_env.rb

@@ -0,0 +1,117 @@
+require "fileutils"
+
+module Portage
+  module Cli
+    # Loads `~/.portage/.env` (or the file PORTAGE_ENV_FILE names) into ENV
+    # before `portage`/`portage-console` run, so shipping address, search
+    # keys and adapter credentials can live in one file next to config.json
+    # instead of a shell profile. A Homebrew install has no repo checkout to
+    # keep a `.env` in, so the user's own Portage directory is the one place
+    # every install method shares.
+    #
+    # Deliberately *not* `./.env` from the working directory: `portage` run
+    # inside a cloned repo would then take that repo's PORTAGE_PROXY/
+    # PORTAGE_PROXY_CA (an intercepting proxy on store traffic), notify
+    # webhooks or store credentials without the user ever choosing them.
+    # PORTAGE_ENV_FILE=.env opts into a project file explicitly.
+    #
+    # Stdlib only (no dotenv gem, so the formula gains no resource). The real
+    # environment always wins, and an empty value is skipped, so a file
+    # copied from .env.example with blanks left in sets nothing.
+    module DotEnv
+      DEFAULT_PATH = File.join(Dir.home, ".portage", ".env").freeze
+      ESCAPES = { "n" => "\n", '"' => '"', "\\" => "\\" }.freeze
+      LINE = /\A\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\z/
+
+      class << self
+        # The file #load read in this process, for `portage doctor`.
+        attr_reader :loaded_path
+      end
+
+      # @return [String, nil] the path that was loaded, if any.
+      def self.load(path: ENV.fetch("PORTAGE_ENV_FILE", nil) || DEFAULT_PATH, env: ENV)
+        path = File.expand_path(path)
+        return unless File.file?(path) && File.readable?(path)
+
+        parse(File.read(path)).each { |key, value| env[key] = value unless env.key?(key) }
+        @loaded_path = path
+      end
+
+      # @return [Hash{String => String}] non-empty assignments, in file order.
+      def self.parse(text)
+        text.each_line.filter_map do |line|
+          match = LINE.match(line.chomp)
+          next unless match
+
+          value = unquote(match[2])
+          [match[1], value] unless value.empty?
+        end.to_h
+      end
+
+      # `"..."` (with \n, \" and \\ escapes) or `'...'` (literal); unquoted
+      # values drop a trailing ` # comment`.
+      def self.unquote(raw)
+        case raw
+        when /\A"((?:[^"\\]|\\.)*)"/
+          Regexp.last_match(1).gsub(/\\([n"\\])/) { ESCAPES.fetch(Regexp.last_match(1)) }
+        when /\A'([^']*)'/ then Regexp.last_match(1)
+        else raw.sub(/\s+#.*\z/, "").strip
+        end
+      end
+      private_class_method :unquote
+
+      # Writes `assignments` into the `.env` file at `path` (creating
+      # `~/.portage` if needed), for `portage setup`'s wizard
+      # (docs/plans/buy-skill-and-local-browser.md Phase 4) — the only
+      # writer this file has had until now. A key already on a line is
+      # replaced in place, so a re-run never duplicates it and every other
+      # line (unrelated keys, comments, blank lines) is left exactly as it
+      # was; a key that isn't already there is appended. `assignments`
+      # values are always double-quoted on write (never bare) so a value
+      # with a space — a street address — round-trips through #parse
+      # correctly.
+      #
+      # Mode 0600 from the very first byte, whether or not the file existed
+      # before: an existing file is chmod'd 600 *before* anything is
+      # written into it (an existing file could already be world-readable),
+      # and a new file is opened with mode 0600 baked into its `File.open`
+      # call — never `File.write` then `File.chmod`, which briefly leaves a
+      # brand-new file (holding a shipping address or a search API key) at
+      # the process umask's permissions, e.g. 0644, before the chmod lands.
+      #
+      # @param assignments [Hash{String => String}]
+      # @return [String] the path written.
+      def self.update!(assignments, path: ENV.fetch("PORTAGE_ENV_FILE", nil) || DEFAULT_PATH)
+        path = File.expand_path(path)
+        remaining = assignments.dup
+        lines = existing_lines(path).map { |line| replace_assignment(line, remaining) }
+        remaining.each { |key, value| lines << assignment_line(key, value) }
+
+        FileUtils.mkdir_p(File.dirname(path))
+        File.chmod(0o600, path) if File.file?(path)
+        File.open(path, File::WRONLY | File::CREAT | File::TRUNC, 0o600) { |f| f.write(lines.join) }
+        File.chmod(0o600, path)
+        path
+      end
+
+      def self.existing_lines(path)
+        File.file?(path) ? File.readlines(path) : []
+      end
+      private_class_method :existing_lines
+
+      def self.replace_assignment(line, remaining)
+        match = LINE.match(line.chomp)
+        return line unless match && remaining.key?(match[1])
+
+        assignment_line(match[1], remaining.delete(match[1]))
+      end
+      private_class_method :replace_assignment
+
+      def self.assignment_line(key, value)
+        escaped = value.to_s.gsub("\\") { "\\\\" }.gsub('"') { "\\\"" }
+        "#{key}=\"#{escaped}\"\n"
+      end
+      private_class_method :assignment_line
+    end
+  end
+end