portage-cli 0.7.4 → 0.8.0

data/lib/portage/cli/find.rb

@@ -3,9 +3,12 @@ require "portage/ucp"
 require "portage/ucp/client"
 
 require_relative "search_backends"
+require_relative "offer_sources"
+require_relative "agent_profile_url"
 require_relative "probe_cache"
 require_relative "decisions"
 require_relative "user_agent"
+require_relative "handoff_only"
 
 module Portage
   module Cli
@@ -29,33 +32,52 @@ module Portage
 
       # @param max_price [Integer, nil] minor units, matching the protocol's
       #   own money representation — the CLI converts from major units.
-      def initialize(query:, limit: MAX_PROBES, max_price: nil, backends: nil, cache: nil, throttle: THROTTLE)
+      # @param offer_sources [Array<#offers>, nil] a second kind of backend
+      #   (OfferSources) that answers offers directly, with no manifest
+      #   probe of its own — see #call. nil (the default) is
+      #   OfferSources.default.
+      # @param handoff_only [HandoffOnly, nil] Tier C's host list
+      #   (docs/plans/buy-skill-and-local-browser.md Phase 5) — nil (the
+      #   default) is the real ~/.portage/config.json-backed one. A
+      #   candidate on it is never probed (see #call): it still surfaces as
+      #   a candidate, marked `handoff_only: true`, so an agent can list it
+      #   ("Amazon also sells this") without this process ever fetching it.
+      def initialize(query:, limit: MAX_PROBES, max_price: nil, backends: nil, cache: nil, throttle: THROTTLE,
+                     offer_sources: nil, handoff_only: nil)
         @query = query.to_s
         @limit = [limit, MAX_PROBES].min
         @max_price = max_price
         @backends = backends || SearchBackends.default
         @cache = cache || ProbeCache.new
         @throttle = throttle
+        @offer_sources = offer_sources || OfferSources.default
+        @handoff_only = handoff_only || HandoffOnly.new
       end
 
       def call
         return report(message: "Nothing to search for — pass --query.") if @query.strip.empty?
 
         candidates = candidate_origins
-        return report(candidates: candidates, message: no_candidates_message) if candidates.empty?
+        probed, stores = probe_candidates(candidates)
+        sourced = source_offers
+        return report(candidates: candidates, message: no_candidates_message) if nothing_to_go_on?(candidates, sourced)
 
-        stores = probe(candidates)
-        offers = rank(stores.flat_map { |store| offers_for(store) })
-        report(candidates: candidates, stores: stores.map { |s| s.slice(:origin, :source, :checkout) },
-               offers: offers, message: summary(candidates, stores, offers))
+        offers = rank(sourced + probed.flat_map { |store| offers_for(store) })
+        report(candidates: candidates, stores: store_summaries(stores), offers: offers,
+               message: summary(candidates, stores, offers))
       rescue Portage::Ucp::Client::MissingAgentProfileError
-        report(candidates: candidates, stores: stores.map { |s| s.slice(:origin, :source, :checkout) },
+        report(candidates: candidates, stores: store_summaries(stores),
                message: "Set PORTAGE_AGENT_PROFILE to a URL that describes this agent — each store " \
                         "verifies it before answering a catalog search.")
       end
 
       private
 
+      # Neither the URL backends nor any OfferSource found anything to
+      # probe or rank — split out of #call to keep its own branching under
+      # the complexity budget.
+      def nothing_to_go_on?(candidates, sourced) = candidates.empty? && sourced.empty?
+
       # --- Step 1: ask the backends who might sell this ---
 
       def candidate_origins
@@ -78,7 +100,44 @@ module Portage
         existing = seen[uri.host]
         return if existing && !upgradable?(existing, uri)
 
-        seen[uri.host] = { origin: origin_of(uri), source: existing ? existing[:source] : backend.name }
+        seen[uri.host] = { origin: origin_of(uri), source: existing ? existing[:source] : backend.name,
+                           handoff_only: @handoff_only.host?(uri.host) }
+      end
+
+      # Tier C candidates never reach #probe — no UCP request, ever — but
+      # still surface in the report's `stores`, `checkout: false`.
+      def handoff_only_stores(candidates)
+        candidates.map { |c| c.merge(checkout: false) }
+      end
+
+      # Splits candidates into the ones #probe actually fetches and the
+      # Tier C ones that never touch the network — kept out of #call to
+      # stay under its own complexity budget.
+      # @return [Array(Array<Hash>, Array<Hash>)] probed stores (the ones
+      #   with a live `session`, so #offers_for can ask them what they
+      #   stock), and every store for the report (probed + hand-off-only).
+      def probe_candidates(candidates)
+        probeable, deferred = candidates.partition { |c| !c[:handoff_only] }
+        probed = probe(probeable)
+        [probed, probed + handoff_only_stores(deferred)]
+      end
+
+      def store_summaries(stores)
+        stores.map { |s| s.slice(:origin, :source, :checkout, :handoff_only) }
+      end
+
+      # --- Step 1b: ask any OfferSources directly, no probe needed ---
+
+      # Each source already returns Find#offer-shaped hashes (store:/
+      # source:/checkout:/product_id:/title:/amount:/currency:/url:) and
+      # swallows its own failures, so nothing here needs the try/rescue
+      # #urls_from gives the URL backends. --max-price applies here exactly
+      # as it does to a probed store's offers in #offer.
+      def source_offers
+        offers = @offer_sources.flat_map do |source|
+          source.offers(@query, limit: PER_STORE_RESULTS, context: BuyerContext.from_env)
+        end
+        offers.reject { |offer| over_max_price?(offer[:amount]) }
       end
 
       def upgradable?(existing, uri)
@@ -156,18 +215,21 @@ module Portage
       # before answering any call (see Transports::Http) — the own-store
       # loopback path ignores it harmlessly.
       def agent_meta
-        { agent_profile: ENV.fetch("PORTAGE_AGENT_PROFILE", nil) }
+        { agent_profile: AgentProfileUrl.resolve }
       end
 
       def offer(store, product)
         amount, currency = price_of(product)
-        return nil if @max_price && amount && amount > @max_price
+        return nil if over_max_price?(amount)
 
         { store: store[:origin], source: store[:source], checkout: store[:checkout],
           product_id: field(product, "id"), title: field(product, "title"),
           amount: amount, currency: currency, url: field(product, "url") }
       end
 
+      # An unpriced offer stays in: no price isn't the same as too dear.
+      def over_max_price?(amount) = @max_price && amount && amount > @max_price
+
       # Buyable first, then cheapest, then unpriced (see Decisions.rank —
       # core's Support::OfferRanking, the rule portage-ucp-decision's
       # OfferRanking wraps), so an agent loop ranking its own candidate list
@@ -213,7 +275,7 @@ module Portage
         names = @backends.map(&:name)
         return no_backends_message if names.empty?
 
-        "No candidate stores came back from #{names.join(', ')} for \"#{@query}\"."
+        "No candidate stores came back from #{names.join(', ')} for \"#{@query}\".#{keyed_backend_hint}"
       end
 
       def no_backends_message
@@ -221,8 +283,30 @@ module Portage
           "or list stores in ~/.portage/stores.yml."
       end
 
+      # DuckDuckGo's free Instant Answer API is the keyless default, but it
+      # only answers *entity* queries ("burton snowboards" → burton.com) —
+      # open-ended shopping terms like "coffee" or "iphone" resolve to
+      # nothing, every time, regardless of what those stores actually stock
+      # (see SearchBackends::DuckDuckGo's own comment). Names alone (e.g.
+      # `no_candidates_message`) don't say *why* nothing came back, so this
+      # spells out the fix rather than leaving the caller to guess whether
+      # it's a network problem or a backend limitation. Only fires when no
+      # keyed backend (Brave/Google CSE) ran alongside it — one of those
+      # already covered the query with real web search, so there's nothing
+      # to suggest.
+      def keyed_backend_hint
+        return "" unless SearchBackends.only_duckduckgo?(@backends)
+
+        " DuckDuckGo's free API only resolves specific brand/product names, not open-ended search — " \
+          "set BRAVE_SEARCH_API_KEY or GOOGLE_CSE_KEY/GOOGLE_CSE_CX for real web search " \
+          "(see `portage doctor`)."
+      end
+
       def summary(candidates, stores, offers)
-        return "Found #{offers.length} offer(s) across #{stores.length} UCP store(s)." if offers.any?
+        # Counted from the offers, not `stores`: an OfferSource's offers
+        # come from stores that were never probed.
+        selling = offers.map { |o| o[:store] }.uniq.length
+        return "Found #{offers.length} offer(s) across #{selling} store(s)." if offers.any?
         return "#{stores.length} store(s) speak UCP but none stock \"#{@query}\"." if stores.any?
 
         "Checked #{candidates.length} store(s); none of them speak UCP."

data/lib/portage/cli/handoff_agents.rb

@@ -0,0 +1,186 @@
+require "json"
+require "net/http"
+require "open3"
+require "timeout"
+require "uri"
+require "portage/ucp/support/connection"
+require_relative "config"
+require_relative "user_agent"
+
+module Portage
+  module Cli
+    # `--handoff-target agent:<name>` (docs/plans/buy-skill-and-local-browser.md
+    # Phase 5, decision 3): looks up a named agent in
+    # ~/.portage/config.json's `handoff_agents` and, only when it's
+    # explicitly `"approved": true`, hands it the same JSON payload
+    # `--notify-webhook` sends (Buy#handoff_notify_payload) — checkout URL
+    # plus the approved cart summary (items, qty, total, store). No
+    # credentials, payment tokens or shipping details beyond what the
+    # checkout URL already holds ever go into that payload.
+    #
+    # Config shape:
+    #   "handoff_agents": {
+    #     "openclaw": { "command": ["openclaw", "handoff"], "approved": true },
+    #     "storefront": { "webhook": "https://example.com/hooks/handoff", "approved": true }
+    #   }
+    #
+    # An entry with no "approved": true (or no entry at all) is never
+    # invoked — #lookup returns nil, and the caller (Buy) reports why and
+    # falls back to `print` behavior.
+    class HandoffAgents
+      CONFIG_KEY = "handoff_agents".freeze
+
+      def initialize(config: Config.load)
+        @config = config
+      end
+
+      # @return [#call, nil] an object responding to #call(payload) that
+      #   returns nil on success or a delivery-failure message — nil when
+      #   `name` has no config entry, or the entry isn't approved.
+      def lookup(name)
+        entry = agents[name.to_s]
+        return nil unless entry.is_a?(Hash) && entry["approved"] == true
+
+        return Command.new(entry["command"]) if entry["command"]
+        return Webhook.new(entry["webhook"]) if entry["webhook"]
+
+        nil
+      end
+
+      private
+
+      def agents
+        raw = @config.get(CONFIG_KEY)
+        raw.is_a?(Hash) ? raw : {}
+      end
+
+      # Runs the configured argv (never a shell string — merchant-controlled
+      # text never reaches a shell), with the payload as JSON on stdin, a
+      # scrubbed environment (only ENV_ALLOWLIST passed through —
+      # PORTAGE_*, API keys and every other ambient var are dropped), and a
+      # hard timeout that kills the process on expiry. A non-zero exit
+      # means "not delivered", same posture as Notifier's non-2xx.
+      class Command
+        TIMEOUT = 30
+        # Grace period between SIGTERM and SIGKILL once TIMEOUT is hit — long
+        # enough for a well-behaved process to notice and exit, short enough
+        # that a hung/ignoring one doesn't stall the hand-off much further.
+        KILL_GRACE = 2
+        BODY_EXCERPT = 200
+        # Only the basics a well-behaved subprocess needs to run at all —
+        # never PORTAGE_*, a payment/API key, or a shipping variable.
+        ENV_ALLOWLIST = %w[PATH HOME LANG LC_ALL TERM TMPDIR TZ].freeze
+
+        def initialize(argv)
+          @argv = Array(argv).map(&:to_s)
+        end
+
+        def call(payload)
+          return "agent command is empty" if @argv.empty?
+
+          stdout, stderr, status = run(payload)
+          return nil if status&.success?
+
+          "agent command exited #{status&.exitstatus.inspect}: #{excerpt(stderr, stdout)}"
+        rescue Timeout::Error
+          "agent command timed out after #{TIMEOUT}s"
+        rescue StandardError => e
+          "agent command failed: #{e.message}"
+        end
+
+        private
+
+        # Reads stdout/stderr on their own threads — same posture as
+        # Open3.capture3 — so a child that fills the stderr pipe while this
+        # blocks reading stdout to EOF (or vice versa) can't deadlock the
+        # hand-off. Open3.popen3's block form always joins `wait_thr` in its
+        # own `ensure` once this block returns, so a timeout has to actually
+        # kill the child (and wait for it to die) *before* returning from
+        # here — otherwise that `ensure` blocks forever on a process nothing
+        # is waiting on anymore, and the TIMEOUT constant does nothing.
+        def run(payload)
+          env = ENV.to_h.slice(*ENV_ALLOWLIST)
+          out = nil
+          err = nil
+          status = nil
+          Open3.popen3(env, *@argv, unsetenv_others: true) do |stdin, stdout, stderr, wait_thr|
+            write_and_close(stdin, payload)
+            out_thread = Thread.new { stdout.read.to_s }
+            err_thread = Thread.new { stderr.read.to_s }
+            status = wait_with_timeout(wait_thr)
+            out = out_thread.value
+            err = err_thread.value
+          end
+          [out, err, status]
+        end
+
+        def wait_with_timeout(wait_thr)
+          Timeout.timeout(TIMEOUT) { wait_thr.value }
+        rescue Timeout::Error
+          kill_and_wait(wait_thr)
+          raise
+        end
+
+        # SIGTERM first, SIGKILL after KILL_GRACE if it's still alive —
+        # either way this doesn't return until `wait_thr.value` resolves, so
+        # the process is confirmed dead (and reaped) before #run's caller
+        # ever gets to leave the `Open3.popen3` block.
+        def kill_and_wait(wait_thr)
+          Process.kill("TERM", wait_thr.pid)
+          Timeout.timeout(KILL_GRACE) { wait_thr.value }
+        rescue Errno::ESRCH
+          nil
+        rescue Timeout::Error
+          kill_quietly(wait_thr.pid)
+          wait_thr.value
+        end
+
+        def kill_quietly(pid)
+          Process.kill("KILL", pid)
+        rescue Errno::ESRCH
+          nil
+        end
+
+        def write_and_close(stdin, payload)
+          stdin.write(JSON.generate(payload))
+        ensure
+          stdin.close
+        end
+
+        def excerpt(stderr, stdout) = (stderr.to_s.empty? ? stdout.to_s : stderr.to_s).strip[0, BODY_EXCERPT]
+      end
+
+      # POSTs the payload as JSON — https only, same connection helper and
+      # short timeout as Notifier, failures swallowed into the return
+      # value rather than raised.
+      class Webhook
+        TIMEOUT = 10
+        BODY_EXCERPT = 200
+
+        def initialize(url)
+          @url = url.to_s
+        end
+
+        def call(payload)
+          return "agent webhook must be https" unless @url.start_with?("https://")
+
+          response = post(URI(@url), JSON.generate(payload))
+          return nil if response.is_a?(Net::HTTPSuccess)
+
+          "agent webhook answered #{response.code}: #{response.body.to_s[0, BODY_EXCERPT]}"
+        rescue StandardError => e
+          "agent webhook POST failed: #{e.message}"
+        end
+
+        private
+
+        def post(uri, body)
+          Portage::Ucp::Support::Connection.start(uri, route: :notify, open_timeout: TIMEOUT,
+                                                       read_timeout: TIMEOUT) do |http|
+            http.post(uri.request_uri, body, UserAgent.headers.merge("Content-Type" => "application/json"))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/handoff_only.rb

@@ -0,0 +1,94 @@
+require "uri"
+require_relative "config"
+
+module Portage
+  module Cli
+    # Tier C (docs/plans/buy-skill-and-local-browser.md Phase 5): the single
+    # place that decides whether a host is hand-off only — Amazon (every
+    # marketplace) by default, plus whatever the user adds or removes in
+    # ~/.portage/config.json's "handoff_only_hosts". `Buy#call`, `Find`,
+    # `Index::Builder` and `BrowserImport::Importer` all read this list
+    # (Importer already had an injectable `handoff_only_hosts:` seam for it)
+    # rather than each hard-coding or re-deriving their own — the one
+    # invariant every caller depends on is "never probed, never fetched,
+    # never automated", and that only holds if there's exactly one place
+    # deciding host membership.
+    #
+    # Absent key: the shipped default list. Present key (even an empty
+    # array): the user's list *is* the list — they can drop Amazon entirely
+    # or add other hosts, and #host? only ever consults what #hosts returns.
+    class HandoffOnly
+      CONFIG_KEY = "handoff_only_hosts".freeze
+
+      # Every Amazon marketplace TLD, current as of this phase — the user's
+      # own config.json list can extend or shrink this once it's present.
+      DEFAULT_HOSTS = %w[
+        amazon.com amazon.co.uk amazon.de amazon.fr amazon.it amazon.es amazon.nl amazon.se amazon.pl
+        amazon.com.be amazon.ie amazon.ca amazon.com.mx amazon.com.br amazon.co.jp amazon.in amazon.com.au
+        amazon.sg amazon.ae amazon.sa amazon.eg amazon.com.tr amazon.cn
+      ].freeze
+
+      # Facts, not legal advice (docs/plans/buy-skill-and-local-browser.md
+      # decision 5): what the site's terms say, plus the as-is/no-warranty
+      # line. Reused verbatim by Buy's `handoff_only` report, `doctor`,
+      # `portage setup`'s wizard, and the `buy` skill's own reference doc.
+      LEGAL_NOTICE = "This retailer's terms restrict automated purchasing agents, so Portage opens the page " \
+                     "and you complete the purchase. Portage is open-source software provided as-is, " \
+                     "without warranty.".freeze
+
+      def initialize(config: Config.load)
+        @config = config
+      end
+
+      # @return [Array<String>] lowercase base hosts (no scheme, no path,
+      #   no "www."). A user entry of "www.example.com", "example.com/",
+      #   or "https://www.example.com/s?k=x" all normalize to the same
+      #   "example.com" — the same normalization
+      #   `BrowserImport::Importer`/`Domains` already apply to a history or
+      #   bookmark domain, so a host written any of those ways behaves the
+      #   same everywhere it's checked.
+      def hosts
+        configured = @config.get(CONFIG_KEY)
+        return DEFAULT_HOSTS unless configured.is_a?(Array)
+
+        configured.filter_map { |h| self.class.normalize_entry(h) }
+      end
+
+      # @param host [String, nil] a bare hostname, e.g. "www.amazon.co.uk".
+      def host?(host) = self.class.matches_any?(host, hosts)
+
+      # Amazon-ness is a fact about the domain, independent of whatever the
+      # user's own config currently lists (they may have removed it from
+      # #hosts and still be on an amazon.* origin) — used by Buy to decide
+      # whether it knows a search/cart-add URL pattern for this host at all.
+      def self.amazon?(host) = matches_any?(host, DEFAULT_HOSTS)
+
+      def self.matches_any?(host, list)
+        normalized = host.to_s.downcase.strip
+        return false if normalized.empty?
+
+        list.any? { |base| normalized == base || normalized.end_with?(".#{base}") }
+      end
+
+      # A bare host ("amazon.co.uk"), one already carrying "www." ("www.
+      # amazon.co.uk"), or a full URL a user pasted in ("https://www.
+      # amazon.co.uk/s?k=x") — a scheme means this is a URL, parsed for its
+      # host; otherwise it's a bare host/host-with-path, so only the part
+      # before the first "/" or "?" counts.
+      def self.normalize_entry(value)
+        text = value.to_s.strip
+        return nil if text.empty?
+
+        host = text.include?("://") ? uri_host(text) : text[%r{\A[^/?\s]+}]
+        host = host.to_s.downcase.delete_prefix("www.")
+        host.empty? ? nil : host
+      end
+
+      def self.uri_host(text)
+        URI.parse(text).host
+      rescue URI::InvalidURIError
+        nil
+      end
+    end
+  end
+end

data/lib/portage/cli/handoff_reconciler.rb

@@ -8,6 +8,7 @@ require_relative "homepage_fetch"
 require_relative "handoff_spend_mode"
 require_relative "reconcile_notifier"
 require_relative "permissive_authenticator"
+require_relative "handoff_only"
 require_relative "handoff_reconciler/wire_adapters"
 
 module Portage
@@ -48,13 +49,15 @@ module Portage
       def initialize(transaction_log: Portage::Ucp::Support::TransactionLog.new,
                      order_ledger: Portage::Ucp::Support::OrderLedger.new,
                      journal: Portage::Ucp::Journal::PurchaseJournal.new,
-                     notifier: ReconcileNotifier.new, spend_mode: HandoffSpendMode.resolve, clock: -> { Time.now })
+                     notifier: ReconcileNotifier.new, spend_mode: HandoffSpendMode.resolve, clock: -> { Time.now },
+                     handoff_only: nil)
         @transaction_log = transaction_log
         @order_ledger = order_ledger
         @journal = journal
         @notifier = notifier
         @spend_mode = spend_mode
         @clock = clock
+        @handoff_only = handoff_only || HandoffOnly.new
       end
 
       # @param record [Hash] a TransactionLog record, as returned by
@@ -216,6 +219,17 @@ module Portage
         raise ReconnectError, "no store_url on this record" if store_url.to_s.empty?
 
         uri = URI.parse(store_url)
+        # Defense in depth (docs/plans/buy-skill-and-local-browser.md
+        # Phase 5): nothing in this gem should ever write a hand-off-only
+        # host's checkout into TransactionLog as a pending `settled_by:
+        # "shopper"` record in the first place (Buy's `handoff_only`
+        # outcome never calls #hand_off/#record_pending_handoff at all),
+        # but this is the one place every reconcile path fetches a store
+        # again from a saved record, so it's guarded here too rather than
+        # trusted to stay that way.
+        raise ReconnectError, "#{uri.host} is hand-off only — never reconciled automatically" \
+          if @handoff_only.host?(uri.host)
+
         native_session(uri) || adapter_session(uri) ||
           raise(ReconnectError, "no automated path back into #{store_url}")
       end

data/lib/portage/cli/handoff_target.rb

@@ -0,0 +1,61 @@
+require_relative "config"
+require_relative "setting"
+
+module Portage
+  module Cli
+    # `portage buy --handoff-target` (docs/plans/buy-skill-and-local-browser.md
+    # Phase 5): which of the four hand-off targets a dead-end checkout_url
+    # goes to. Same Setting precedence as CheckoutHandoff/Notifier — a
+    # per-invocation override beats PORTAGE_HANDOFF_TARGET, which beats
+    # ~/.portage/config.json's "handoff_target" — and defaults to "default"
+    # (today's CheckoutHandoff auto-open behavior) when nothing is set at
+    # any level.
+    #
+    # Raises ArgumentError on anything else, so `Cli.run_buy` can treat a
+    # bad value as a usage error the same way it already does for a bad
+    # --min-confidence (see Cli.confidence_check) — built once, up front,
+    # before a checkout is ever attempted.
+    class HandoffTarget
+      ENV_VAR = "PORTAGE_HANDOFF_TARGET".freeze
+      CONFIG_KEY = "handoff_target".freeze
+      DEFAULT = "default".freeze
+      KNOWN_KINDS = %w[default print profile].freeze
+      AGENT_PATTERN = /\Aagent:(.+)\z/
+
+      attr_reader :kind, :agent_name
+
+      def initialize(override: nil, config: Config.load)
+        value = Setting.resolve(override: override, env: ENV_VAR, config: config, config_key: CONFIG_KEY)
+        value = value.to_s.strip
+        parse!(value.empty? ? DEFAULT : value)
+      end
+
+      def default? = @kind == "default"
+      def print? = @kind == "print"
+      def profile? = @kind == "profile"
+      def agent? = @kind == "agent"
+
+      # `agent:<name>`'s own label for a report/dispatch — "agent:<name>"
+      # for an agent target, or just the kind otherwise.
+      def label = agent? ? "agent:#{agent_name}" : @kind
+
+      private
+
+      def parse!(value)
+        if KNOWN_KINDS.include?(value)
+          @kind = value
+          return
+        end
+
+        match = AGENT_PATTERN.match(value)
+        unless match
+          raise ArgumentError, "Unknown --handoff-target #{value.inspect} — use default, print, profile, " \
+                               "or agent:<name>."
+        end
+
+        @kind = "agent"
+        @agent_name = match[1]
+      end
+    end
+  end
+end