portage-cli 0.7.5 → 0.8.0

data/lib/portage/cli/index/sources/browser.rb

@@ -0,0 +1,31 @@
+module Portage
+  module Cli
+    module Index
+      module Sources
+        # A pointer, not a reader. Browser-derived entries come from
+        # `portage browser import` (BrowserImport::Importer, Phase 3), which
+        # writes them straight into Index::Store — `sources: ["history"]`/
+        # `["bookmark"]` — only after the user has seen the list and
+        # approved it. `index build` must never read a browser's history on
+        # its own (that would skip the approval), so this source yields
+        # nothing and isn't in Sources::DEFAULT_NAMES; it stays in the
+        # registry so `portage index sources` still says where browser
+        # entries come from.
+        class Browser
+          def name = "browser"
+
+          def description
+            "Bookmarks/history shop domains — written by `portage browser import` after you approve the list " \
+              "(never read by `index build` itself). Yields nothing here."
+          end
+
+          def source_path = nil
+
+          # `**` accepts (and ignores) the shared Source#candidates(queries:)
+          # interface.
+          def candidates(**) = []
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/index/sources/shopify_catalog.rb

@@ -0,0 +1,82 @@
+require "yaml"
+
+require_relative "../../offer_sources"
+require_relative "../../buyer_context"
+require_relative "../../classifier"
+
+module Portage
+  module Cli
+    module Index
+      module Sources
+        # Harvests merchant origins and product identities from Shopify's
+        # global catalog — the same source `find` already uses live
+        # (OfferSources::ShopifyCatalog), reused here rather than
+        # duplicating its catalog-search/variant-URL logic (docs/plans/
+        # buy-skill-and-local-browser.md Phase 2b: "reuse the Phase 1
+        # ShopifyCatalog code rather than duplicating it").
+        #
+        # One query per top-level taxonomy node by default (21 nodes —
+        # known-stores/categories.yml's own top level, its keyword rather
+        # than its full "Parent > Child" name, so each query is a plain
+        # shopping term like "electronics" or "apparel" instead of a taxon
+        # label). Not per-country: BuyerContext.from_env already applies
+        # whatever PORTAGE_SHIP_COUNTRY/PORTAGE_CURRENCY the user has set,
+        # same as every other catalog call this CLI makes, and fanning the
+        # same 21 queries out across every market on every `index build`
+        # would turn a several-second command into a slow one for no
+        # benefit a single run doesn't already get from the user's own
+        # locale (Open question 1, resolved 2026-09-28).
+        class ShopifyCatalog
+          PER_QUERY_LIMIT = 20
+
+          def initialize(catalog: OfferSources::ShopifyCatalog.new, categories_path: Classifier::KNOWN_PATH)
+            @catalog = catalog
+            @categories_path = categories_path
+          end
+
+          def name = "shopify_catalog"
+
+          def description
+            "Shopify's global catalog search (catalog.shopify.com) — one query per top-level taxonomy " \
+              "node, or --queries FILE. Harvests merchant origins from variants[].url and product " \
+              "identities from the results."
+          end
+
+          def source_path = nil
+
+          # @param queries [Array<String>, nil] nil (the default) runs one
+          #   query per top-level taxonomy node.
+          # @return [Array<Hash>] sightings: origin:, url:, title:, brand:,
+          #   gtin:.
+          def candidates(queries: nil)
+            (queries || default_queries).flat_map { |query| sightings_for(query) }
+          end
+
+          private
+
+          def sightings_for(query)
+            @catalog.offers(query, limit: PER_QUERY_LIMIT, context: BuyerContext.from_env).map do |offer|
+              { origin: offer[:store], url: offer[:url], title: offer[:title], brand: nil, gtin: nil }
+            end
+          rescue StandardError
+            []
+          end
+
+          # The shipped taxonomy's own top-level keyword ("animals",
+          # "apparel", "electronics", ...) rather than its full name — a
+          # single plain word is a better catalog search term than
+          # "Apparel & Accessories" would be.
+          def default_queries
+            data = YAML.safe_load_file(@categories_path)
+            return [] unless data.is_a?(Hash)
+
+            top_level = data.values.reject { |node| node["name"].to_s.include?(">") }
+            top_level.filter_map { |node| Array(node["keywords"]).first }.uniq
+          rescue StandardError
+            []
+          end
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/index/sources/stores_file.rb

@@ -0,0 +1,58 @@
+require "uri"
+
+require_relative "../../search_backends"
+
+module Portage
+  module Cli
+    module Index
+      module Sources
+        # The user's own `~/.portage/stores.yml` (or `PORTAGE_STORES`) —
+        # already-trusted stores they listed themselves. Reuses
+        # SearchBackends::Allowlist's own parsing (env + file, bare URL or
+        # `{url:, categories:}`) rather than re-reading the file, so a
+        # change to that format only has to be made in one place.
+        class StoresFile
+          def initialize(allowlist: SearchBackends::Allowlist.new)
+            @allowlist = allowlist
+          end
+
+          def name = "stores_file"
+
+          def description
+            "Your own ~/.portage/stores.yml / PORTAGE_STORES — stores you've already trusted directly."
+          end
+
+          def source_path = SearchBackends::Allowlist::PATH
+
+          # @return [Array<Hash>] one sighting per store, no product
+          #   identity (stores.yml only ever names a store, never a
+          #   product). `**` accepts (and ignores) the shared
+          #   Source#candidates(queries:) interface — stores.yml has no
+          #   query of its own to run.
+          def candidates(**)
+            @allowlist.stores.filter_map { |entry| sighting_for(entry[:url]) }
+          end
+
+          private
+
+          def sighting_for(url)
+            origin = origin_of(url)
+            return nil unless origin
+
+            { origin: origin, url: nil, title: nil, brand: nil, gtin: nil }
+          end
+
+          def origin_of(url)
+            uri = URI.parse(url.to_s)
+            return nil unless uri.host && uri.scheme.to_s.start_with?("http")
+
+            port = uri.port == uri.default_port ? "" : ":#{uri.port}"
+            "#{uri.scheme}://#{uri.host}#{port}"
+          rescue URI::InvalidURIError
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/index/sources/webmcp_sweep.rb

@@ -0,0 +1,29 @@
+module Portage
+  module Cli
+    module Index
+      module Sources
+        # Would detect each known origin's WebMCP preset by driving a
+        # browser bridge against it (docs/plans/webmcp-universal-outbound.md)
+        # — that bridge doesn't exist yet (it's Phase 6 here / WebMCP Phase
+        # 3-4), so this source skips cleanly rather than failing: it's
+        # listed by `portage index sources` so its eventual arrival is
+        # discoverable, but `candidates` is always empty until a bridge is
+        # wired in.
+        class WebmcpSweep
+          def name = "webmcp_sweep"
+
+          def description
+            "Detects a WebMCP preset per known origin via the browser bridge. No bridge is wired up yet " \
+              "— skips cleanly, yields nothing."
+          end
+
+          def source_path = nil
+
+          # `**` accepts (and ignores) the shared Source#candidates(queries:)
+          # interface, since this source never runs a query of its own.
+          def candidates(**) = []
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/index/sources/wikidata.rb

@@ -0,0 +1,95 @@
+require "net/http"
+require "uri"
+require "json"
+require "timeout"
+require "portage/ucp/support/connection"
+
+require_relative "../../user_agent"
+
+module Portage
+  module Cli
+    module Index
+      module Sources
+        # Wikidata SPARQL for retail chains' and brands' official websites
+        # (CC0). Optional and **off by default** (Sources.default excludes
+        # it) — a live, read-only trial run of this exact query on
+        # 2026-09-28 returned real hits (Venchi, DNS, Fix Price, Avtomir and
+        # more, each with a genuine official-site URL), which is the bar
+        # the plan set for keeping it, but the yield skews toward chains
+        # with no UCP/WebMCP support at all — most of what it finds fails
+        # the `/.well-known/ucp` probe like any other candidate, same as a
+        # web-search backend's own noise. Worth keeping for the rare real
+        # hit, not worth running on every default `index build`.
+        class Wikidata
+          ENDPOINT = "https://query.wikidata.org/sparql".freeze
+          TIMEOUT = 15
+          LIMIT = 200
+
+          # wd:Q507619 = "retail chain". wdt:P279* walks subclasses, so a
+          # supermarket chain or department-store chain tagged as a
+          # subclass of retail chain is included too, not just an exact
+          # match.
+          QUERY = <<~SPARQL
+            SELECT ?item ?itemLabel ?website WHERE {
+              ?item wdt:P31/wdt:P279* wd:Q507619 .
+              ?item wdt:P856 ?website .
+              SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
+            } LIMIT #{LIMIT}
+          SPARQL
+                  .freeze
+
+          def name = "wikidata"
+
+          def description
+            "Wikidata SPARQL for retail chains' official websites (CC0). Off by default (low yield) " \
+              "— opt in with --sources wikidata."
+          end
+
+          def source_path = nil
+
+          # `**` accepts (and ignores) the shared Source#candidates(queries:)
+          # interface — this source runs one fixed SPARQL query, never the
+          # caller's.
+          def candidates(**)
+            rows = Timeout.timeout(TIMEOUT) { fetch }
+            rows.filter_map { |row| sighting_for(row) }
+          rescue StandardError
+            []
+          end
+
+          private
+
+          def fetch
+            uri = URI.parse(ENDPOINT)
+            uri.query = URI.encode_www_form(query: QUERY)
+            headers = UserAgent.headers.merge("Accept" => "application/sparql-results+json")
+            response = Portage::Ucp::Support::Connection.start(
+              uri, route: :search, open_timeout: 5, read_timeout: TIMEOUT
+            ) { |http| http.get(uri.request_uri, headers) }
+            return [] unless response.is_a?(Net::HTTPSuccess)
+
+            Array(JSON.parse(response.body).dig("results", "bindings"))
+          end
+
+          def sighting_for(row)
+            url = row.dig("website", "value")
+            origin = origin_of(url)
+            return nil unless origin
+
+            { origin: origin, url: url, title: nil, brand: row.dig("itemLabel", "value"), gtin: nil }
+          end
+
+          def origin_of(url)
+            uri = URI.parse(url.to_s)
+            return nil unless uri.host && uri.scheme.to_s.start_with?("http")
+
+            port = uri.port == uri.default_port ? "" : ":#{uri.port}"
+            "#{uri.scheme}://#{uri.host}#{port}"
+          rescue URI::InvalidURIError
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/index/sources.rb

@@ -0,0 +1,44 @@
+require_relative "sources/shopify_catalog"
+require_relative "sources/stores_file"
+require_relative "sources/browser"
+require_relative "sources/wikidata"
+require_relative "sources/webmcp_sweep"
+
+module Portage
+  module Cli
+    module Index
+      # One small file per source under index/sources/ — `portage index
+      # sources` lists exactly this registry, so what the command shows and
+      # what `index build` can actually run never drift apart.
+      module Sources
+        ALL = {
+          "shopify_catalog" => -> { ShopifyCatalog.new },
+          "stores_file" => -> { StoresFile.new },
+          "browser" => -> { Browser.new },
+          "wikidata" => -> { Wikidata.new },
+          "webmcp_sweep" => -> { WebmcpSweep.new }
+        }.freeze
+
+        # Runs with no extra opt-in: no bridge required, and no low-yield
+        # trade-off to accept up front. `wikidata` and `webmcp_sweep` still
+        # show up in `portage index sources`, just not run unless named in
+        # `--sources`. `browser` is listed but never a default — it yields
+        # nothing; `portage browser import` writes those entries itself
+        # (see Sources::Browser).
+        DEFAULT_NAMES = %w[shopify_catalog stores_file].freeze
+
+        def self.all = ALL.values.map(&:call)
+
+        def self.default = DEFAULT_NAMES.map { |n| build(n) }
+
+        # @param names [Array<String>]
+        # @return [Array<#name, #description, #source_path, #candidates>]
+        #   unknown names are silently dropped — callers report them
+        #   themselves (see Cli's own option-parsing errors elsewhere).
+        def self.by_name(names) = Array(names).filter_map { |n| build(n) }
+
+        def self.build(name) = ALL[name]&.call
+      end
+    end
+  end
+end

data/lib/portage/cli/index/store.rb

@@ -0,0 +1,109 @@
+require "json"
+require "fileutils"
+require "uri"
+
+module Portage
+  module Cli
+    module Index
+      # `~/.portage/index/stores.json` — one entry per merchant origin
+      # `portage index build`/`refresh`/`add` has found and verified.
+      #
+      # This is untrusted, user-owned data, same posture as `stores.yml`
+      # before it's tagged: never checked into git (docs/plans/
+      # buy-skill-and-local-browser.md Phase 2b), never written into
+      # Portage::Ucp::Policy#merchant_allowlist, and never itself the
+      # "picked a store" a `--yes` buy requires — SearchBackends::Index just
+      # hands back candidate URLs, same as any other search backend, so
+      # `Cli.buy_from_search`'s existing --store/interactive-pick gate
+      # applies to an index-sourced offer exactly as it does to a web
+      # search result (see search_backends_spec.rb/cli_spec.rb for the
+      # specs proving both).
+      #
+      # Entry shape (string keys, JSON on disk): origin, platform,
+      # ucp_version, capabilities (array of "catalog"/"cart"/"checkout"),
+      # webmcp_preset, categories (category id => weight, top 5),
+      # sources (array of source names that found it), last_verified
+      # (unix seconds), handoff_only.
+      class Store
+        PATH = File.join(Dir.home, ".portage", "index", "stores.json").freeze
+
+        def initialize(path: PATH)
+          @path = path
+        end
+
+        def all = entries.values
+
+        def find(origin) = entries[origin]
+
+        # Merges `fields` onto whatever's already there for `origin` (or
+        # starts a fresh entry) — so a second source finding the same store
+        # adds to its `sources`/`categories` rather than clobbering the
+        # first source's findings.
+        def upsert(origin, **fields)
+          existing = entries[origin] || { "origin" => origin }
+          entries[origin] = existing.merge(fields.transform_keys(&:to_s))
+          write
+          entries[origin]
+        end
+
+        # @return [Integer] how many entries (0 or 1 — origins are unique)
+        #   this host's removal actually dropped.
+        def remove(host)
+          before = entries.length
+          entries.reject! { |origin, _| host_of(origin) == host }
+          write
+          before - entries.length
+        end
+
+        # @return [Integer, nil] seconds since the oldest entry was
+        #   verified — nil when the index is empty. `doctor` and `index
+        #   show` use this to say how stale the local index is.
+        def oldest_verified_age(now: Time.now)
+          timestamps = entries.values.filter_map { |e| e["last_verified"] }
+          return nil if timestamps.empty?
+
+          now.to_i - timestamps.min
+        end
+
+        def exists? = File.exist?(@path)
+
+        private
+
+        def host_of(origin)
+          URI.parse(origin).host.to_s
+        rescue URI::InvalidURIError
+          ""
+        end
+
+        def entries
+          @entries ||= read
+        end
+
+        def read
+          return {} unless File.readable?(@path)
+
+          # A business name/title can carry non-ASCII bytes (curly quotes,
+          # ®, accents); reading with the process's default external
+          # encoding (US-ASCII on a bare-minimal LANG, confirmed live in a
+          # sandbox with no locale set) would otherwise raise on the very
+          # first non-ASCII byte and get silently swallowed below, quietly
+          # dropping the whole file's worth of entries — this was caught by
+          # this phase's own live index-build check.
+          parsed = JSON.parse(File.read(@path, encoding: "UTF-8"))
+          parsed.is_a?(Hash) ? parsed : {}
+        rescue StandardError
+          {}
+        end
+
+        # An index that can't be written just means this run's findings
+        # aren't saved — never a failed build.
+        def write
+          FileUtils.mkdir_p(File.dirname(@path))
+          File.write(@path, JSON.generate(@entries))
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/index.rb

@@ -0,0 +1,20 @@
+require_relative "index/store"
+require_relative "index/product_store"
+require_relative "index/known_cache"
+require_relative "index/sources"
+require_relative "index/builder"
+
+module Portage
+  module Cli
+    # `portage index` — a local store and product index the user builds
+    # themselves (docs/plans/buy-skill-and-local-browser.md Phase 2b),
+    # feeding `find` a fresh install's stores.yml can't. Never checked into
+    # git; never trusted the way stores.yml is (see Index::Store's own
+    # comment). Phase 2c adds the repo's own jsdelivr-hosted known-stores
+    # list (Index::KnownCache), fetched under the user's own entries here —
+    # never in place of them.
+    module Index
+      DIR = File.join(Dir.home, ".portage", "index").freeze
+    end
+  end
+end

data/lib/portage/cli/known_stores_url.rb

@@ -0,0 +1,15 @@
+module Portage
+  module Cli
+    # Where the repo's own `known-stores/{stores,products}.json` (docs/plans/
+    # buy-skill-and-local-browser.md Phase 2c) are published for a fresh
+    # install to fetch: the same jsdelivr `@main` channel AgentProfileUrl
+    # already uses for the agent profile. Free GitHub, no Actions — a new
+    # entry reaches every install on its next refresh as soon as a PR
+    # merges, with no gem/brew release to wait on.
+    module KnownStoresUrl
+      BASE = "https://cdn.jsdelivr.net/gh/tomtom87/Portage@main/portage-cli/known-stores".freeze
+      STORES = "#{BASE}/stores.json".freeze
+      PRODUCTS = "#{BASE}/products.json".freeze
+    end
+  end
+end