portage-cli 0.9.0 → 0.11.0

data/lib/portage/cli/index/store.rb

@@ -1,12 +1,13 @@
-require "json"
-require "fileutils"
 require "uri"
+require_relative "database"
 
 module Portage
   module Cli
     module Index
-      # `~/.portage/index/stores.json` — one entry per merchant origin
-      # `portage index build`/`refresh`/`add` has found and verified.
+      # The `stores` table of `~/.portage/index/index.sqlite3` (Index::Database;
+      # `stores.json` before docs/plans/local-catalogue.md Phase 1) — one
+      # entry per merchant origin `portage index build`/`refresh`/`add` has
+      # found and verified.
       #
       # This is untrusted, user-owned data, same posture as `stores.yml`
       # before it's tagged: never checked into git (docs/plans/
@@ -19,7 +20,7 @@ module Portage
       # search result (see search_backends_spec.rb/cli_spec.rb for the
       # specs proving both).
       #
-      # Entry shape (string keys, JSON on disk): origin, platform,
+      # Entry shape (string keys, a JSON object per row): origin, platform,
       # ucp_version, capabilities (array of "catalog"/"cart"/"checkout"),
       # webmcp_preset, categories (category id => weight, top 5),
       # sources (array of source names that found it), last_verified
@@ -27,45 +28,49 @@ module Portage
       class Store
         PATH = File.join(Dir.home, ".portage", "index", "stores.json").freeze
 
+        # @param path [String] where the legacy stores.json lives (or would
+        #   live) — the database sits beside it, and a stores.json found
+        #   there is imported on first open.
         def initialize(path: PATH)
-          @path = path
+          @db = Database.new(path: Database.path_for(path))
         end
 
-        def all = entries.values
+        def all = @db.entries("stores").values
 
-        def find(origin) = entries[origin]
+        def find(origin) = @db.get("stores", origin)
 
         # Merges `fields` onto whatever's already there for `origin` (or
         # starts a fresh entry) — so a second source finding the same store
         # adds to its `sources`/`categories` rather than clobbering the
         # first source's findings.
         def upsert(origin, **fields)
-          existing = entries[origin] || { "origin" => origin }
-          entries[origin] = existing.merge(fields.transform_keys(&:to_s))
-          write
-          entries[origin]
+          @db.transaction do
+            existing = @db.get("stores", origin) || { "origin" => origin }
+            existing.merge(fields.transform_keys(&:to_s)).tap { |merged| @db.put("stores", origin, merged) }
+          end
         end
 
         # @return [Integer] how many entries (0 or 1 — origins are unique)
         #   this host's removal actually dropped.
         def remove(host)
-          before = entries.length
-          entries.reject! { |origin, _| host_of(origin) == host }
-          write
-          before - entries.length
+          @db.transaction do
+            doomed = @db.entries("stores").keys.select { |origin| host_of(origin) == host }
+            doomed.each { |origin| @db.delete("stores", origin) }
+            doomed.length
+          end
         end
 
         # @return [Integer, nil] seconds since the oldest entry was
         #   verified — nil when the index is empty. `doctor` and `index
         #   show` use this to say how stale the local index is.
         def oldest_verified_age(now: Time.now)
-          timestamps = entries.values.filter_map { |e| e["last_verified"] }
+          timestamps = all.filter_map { |e| e["last_verified"] }
           return nil if timestamps.empty?
 
           now.to_i - timestamps.min
         end
 
-        def exists? = File.exist?(@path)
+        def exists? = @db.exists?
 
         private
 
@@ -74,35 +79,6 @@ module Portage
         rescue URI::InvalidURIError
           ""
         end
-
-        def entries
-          @entries ||= read
-        end
-
-        def read
-          return {} unless File.readable?(@path)
-
-          # A business name/title can carry non-ASCII bytes (curly quotes,
-          # ®, accents); reading with the process's default external
-          # encoding (US-ASCII on a bare-minimal LANG, confirmed live in a
-          # sandbox with no locale set) would otherwise raise on the very
-          # first non-ASCII byte and get silently swallowed below, quietly
-          # dropping the whole file's worth of entries — this was caught by
-          # this phase's own live index-build check.
-          parsed = JSON.parse(File.read(@path, encoding: "UTF-8"))
-          parsed.is_a?(Hash) ? parsed : {}
-        rescue StandardError
-          {}
-        end
-
-        # An index that can't be written just means this run's findings
-        # aren't saved — never a failed build.
-        def write
-          FileUtils.mkdir_p(File.dirname(@path))
-          File.write(@path, JSON.generate(@entries))
-        rescue StandardError
-          nil
-        end
       end
     end
   end

data/lib/portage/cli/index.rb

@@ -1,5 +1,6 @@
 require_relative "index/store"
 require_relative "index/product_store"
+require_relative "index/entry_product"
 require_relative "index/known_cache"
 require_relative "index/sources"
 require_relative "index/builder"

data/lib/portage/cli/offer_sources.rb

@@ -49,6 +49,19 @@ module Portage
 
       def self.retail_handoff_host?(host) = HandoffOnly.matches_any?(host, RETAIL_HANDOFF_HOSTS)
 
+      # Adds the UCP Product wire hash to an offer as `product` (docs/plans/
+      # local-catalogue.md Phase 3), as the store served it, with `media`
+      # cut to the first image. Shared by Find and ShopifyCatalog so both
+      # trim the same way. Anything that isn't a non-empty hash leaves the
+      # offer as it was: the field is left out, not faked.
+      def self.with_product(offer, product)
+        return offer unless product.is_a?(Hash) && !product.empty?
+
+        media = product["media"]
+        product = product.merge("media" => media.first(1)) if media.is_a?(Array) && media.length > 1
+        offer.merge(product: product)
+      end
+
       # Shared by every retailer source below: an offer's `store` is the
       # origin of the item URL the API itself returned, same idea as
       # ShopifyCatalog#origin_of — kept as one module method instead of
@@ -133,8 +146,10 @@ module Portage
           return nil unless origin
 
           amount, currency = price_of(product)
-          { store: origin, source: name, checkout: nil, product_id: variant["id"], title: product["title"],
-            amount: amount, currency: currency, url: variant["url"] }
+          OfferSources.with_product(
+            { store: origin, source: name, checkout: nil, product_id: variant["id"], title: product["title"],
+              amount: amount, currency: currency, url: variant["url"] }, product
+          )
         end
 
         def origin_of(url)

data/lib/portage/cli/version.rb

@@ -1,5 +1,5 @@
 module Portage
   module Cli
-    VERSION = "0.9.0".freeze
+    VERSION = "0.11.0".freeze
   end
 end

data/lib/portage/cli.rb

@@ -15,6 +15,7 @@ require_relative "cli/browser_profile"
 require_relative "cli/buy"
 require_relative "cli/find"
 require_relative "cli/compare"
+require_relative "cli/check"
 require_relative "cli/history"
 require_relative "cli/quotes"
 require_relative "cli/money"
@@ -56,6 +57,7 @@ module Portage
              portage find --query "..." [--max-price N] [--limit N] [--json]
              portage compare <url> --product-id ID [--id VALUE ...] [--results N]
                                     [--max-price N] [--json]
+             portage check <url> [--json]
              portage pick [--search LAST|SEARCH_ID] [--via auto|tty|agent] [--json]
                           [--choose REF | --compare REF | --view REF]
              portage approve QUOTE_ID [--via auto|tty|agent] [--relayed-yes | --view] [--json]
@@ -77,8 +79,9 @@ module Portage
              portage orders reconcile [--checkout ID] [--json]
              portage index build [--sources a,b] [--queries FILE] [--dry-run] [--export DIR] [--json]
              portage index refresh [--sources a,b] [--queries FILE] [--dry-run] [--export DIR] [--json]
-             portage index show [--stores|--products] [--json]
-             portage index add <url> [--json]
+             portage index show [--stores|--products [--page N] [--per-page N]] [--json]
+             portage index search QUERY [--category ID] [--store HOST] [--limit N] [--json]
+             portage index add <url> [--crawl] [--json]
              portage index remove <host> [--json]
              portage index sources [--json]
              portage browser import [--browser chrome|edge|brave|arc|firefox|safari] [--profile-root DIR]
@@ -93,13 +96,13 @@ module Portage
              portage generate agent-profile [--out FILE] [--key-out FILE] [--rotate]
              portage --version
 
-           proxy flags (buy/find/compare/doctor/payment enroll):
+           proxy flags (buy/find/compare/check/doctor/payment enroll):
              [--proxy URL] [--proxy-mode forward|gateway] [--proxy-header "Name: value"]
              [--no-proxy HOSTS] [--proxy-route ROUTE=URL|direct] [--proxy-chain URL,URL,...]
              [--proxy-passthrough HEADER] [--proxy-ca FILE] [--no-env-proxy]
     USAGE
 
-    COMMANDS = { "buy" => :run_buy, "find" => :run_find, "compare" => :run_compare,
+    COMMANDS = { "buy" => :run_buy, "find" => :run_find, "compare" => :run_compare, "check" => :run_check,
                  "pick" => :run_pick, "approve" => :run_approve,
                  "history" => :run_history, "payment" => :run_payment, "policy" => :run_policy,
                  "orders" => :run_orders, "index" => :run_index, "browser" => :run_browser,
@@ -207,6 +210,54 @@ module Portage
     def self.to_minor_units(major) = (major * 100).round
     private_class_method :to_minor_units
 
+    # --- check ---
+
+    # `portage check <url>` — can Portage buy from this store, and how?
+    # Exit 0 for `automated`/`webmcp`, 1 for anything else (and for a bad
+    # or missing URL, with the usage on stderr).
+    def self.run_check(argv)
+      opts = parse_check_options(argv)
+      return 1 unless opts
+      return 1 unless apply_proxy_settings(opts[:proxy])
+
+      report = Check.new(opts[:url]).call
+      puts opts[:json] ? JSON.pretty_generate(report) : format_check(report)
+      Check::USABLE_VERDICTS.include?(report[:verdict]) ? 0 : 1
+    end
+    private_class_method :run_check
+
+    def self.parse_check_options(argv)
+      opts = { proxy: {} }
+      OptionParser.new do |parser|
+        parser.on("--json") { opts[:json] = true }
+        ProxySettings.add_options(parser, opts[:proxy])
+      end.parse!(argv)
+      opts[:url] = argv.shift.to_s.strip
+      return opts if !opts[:url].empty? && argv.empty? && valid_check_url?(opts[:url])
+
+      warn USAGE
+      nil
+    end
+    private_class_method :parse_check_options
+
+    def self.valid_check_url?(url)
+      uri = URI.parse(url =~ %r{\Ahttps?://}i ? url : "https://#{url}")
+      uri.is_a?(URI::HTTP) && !uri.host.to_s.empty?
+    rescue URI::InvalidURIError
+      false
+    end
+    private_class_method :valid_check_url?
+
+    def self.format_check(report)
+      lines = ["#{report[:url]}: #{report[:verdict]}"]
+      lines << "  platform: #{report[:platform]}" if report[:platform]
+      lines << "  webmcp: #{report[:webmcp][:status]}#{" (#{report[:webmcp][:reason]})" if report[:webmcp][:reason]}"
+      lines << "  next: #{report[:next_step]}"
+      lines << "  index: `#{report[:index_hint]}` adds its catalogue to your local index" if report[:index_hint]
+      lines.join("\n")
+    end
+    private_class_method :format_check
+
     # --- compare ---
 
     def self.run_compare(argv)
@@ -1312,6 +1363,7 @@ module Portage
       "build" => ->(argv) { run_index_build(argv, refresh: false) },
       "refresh" => ->(argv) { run_index_build(argv, refresh: true) },
       "show" => ->(argv) { run_index_show(argv) },
+      "search" => ->(argv) { run_index_search(argv) },
       "add" => ->(argv) { run_index_add(argv) },
       "remove" => ->(argv) { run_index_remove(argv) },
       "sources" => ->(argv) { run_index_sources(argv) }
@@ -1370,41 +1422,119 @@ module Portage
     end
     private_class_method :format_index_build
 
+    INDEX_SHOW_PER_PAGE = 50
+
     def self.run_index_show(argv)
-      opts = { kind: nil, json: false }
+      opts = { kind: nil, json: false, page: 1, per_page: INDEX_SHOW_PER_PAGE }
       OptionParser.new do |parser|
         parser.on("--stores") { opts[:kind] = "stores" }
         parser.on("--products") { opts[:kind] = "products" }
+        parser.on("--page N", Integer) { |v| opts[:page] = [v, 1].max }
+        parser.on("--per-page N", Integer) { |v| opts[:per_page] = [v, 1].max }
         parser.on("--json") { opts[:json] = true }
       end.parse!(argv)
 
-      result = { stores: opts[:kind] == "products" ? [] : Index::Store.new.all,
-                 products: opts[:kind] == "stores" ? [] : Index::ProductStore.new.all }
+      result = opts[:kind] == "products" ? index_products_page(opts) : index_show_all(opts[:kind])
       puts opts[:json] ? JSON.pretty_generate(result) : format_index_show(result)
       0
     end
     private_class_method :run_index_show
 
+    def self.index_show_all(kind)
+      { stores: kind == "products" ? [] : Index::Store.new.all, products: kind == "stores" ? [] : Index::ProductStore.new.all }
+    end
+    private_class_method :index_show_all
+
+    # `--products` pages (docs/plans/local-catalogue.md Phase 2): a crawled
+    # index can hold thousands.
+    def self.index_products_page(opts)
+      products = Index::ProductStore.new
+      { stores: [], products: products.page(opts[:page], per_page: opts[:per_page]), page: opts[:page],
+        per_page: opts[:per_page], products_total: products.count }
+    end
+    private_class_method :index_products_page
+
     def self.format_index_show(result)
-      lines = ["Stores:"]
-      result[:stores].each { |s| lines << "  #{s['origin']} (#{Array(s['sources']).join(', ')})" }
-      lines << "(none)" if result[:stores].empty?
+      paged = result.key?(:page)
+      lines = paged ? [] : index_store_lines(result[:stores])
       lines << "Products:"
       result[:products].each { |p| lines << "  #{p['title'] || p['key']}" }
       lines << "(none)" if result[:products].empty?
+      lines << index_page_line(result) if paged
       lines.join("\n")
     end
     private_class_method :format_index_show
 
+    def self.index_store_lines(stores)
+      lines = ["Stores:"] + stores.map { |s| "  #{s['origin']} (#{Array(s['sources']).join(', ')})" }
+      stores.empty? ? lines + ["(none)"] : lines
+    end
+    private_class_method :index_store_lines
+
+    def self.index_page_line(result)
+      first = ((result[:page] - 1) * result[:per_page]) + 1
+      last = first + result[:products].length - 1
+      return "Page #{result[:page]} is past the end (#{result[:products_total]} product(s))." if last < first
+
+      "Showing #{first}-#{last} of #{result[:products_total]}. Next: --page #{result[:page] + 1}" \
+        "#{" --per-page #{result[:per_page]}" unless result[:per_page] == INDEX_SHOW_PER_PAGE}"
+    end
+    private_class_method :index_page_line
+
+    # `portage index search` (docs/plans/local-catalogue.md Phase 2): local
+    # index entries only, no request. Exits 1 with no hits.
+    def self.run_index_search(argv)
+      opts = { category: nil, store: nil, limit: 20, json: false }
+      OptionParser.new do |parser|
+        parser.on("--category ID") { |v| opts[:category] = v }
+        parser.on("--store HOST") { |v| opts[:store] = v }
+        parser.on("--limit N", Integer) { |v| opts[:limit] = [v, 1].max }
+        parser.on("--json") { opts[:json] = true }
+      end.parse!(argv)
+      query = argv.join(" ").strip
+      return (warn USAGE) || 1 if query.empty?
+
+      result = index_search(query, opts)
+      puts opts[:json] ? JSON.pretty_generate(result) : format_index_search(result)
+      result[:products].empty? ? 1 : 0
+    end
+    private_class_method :run_index_search
+
+    def self.index_search(query, opts)
+      products = Index::ProductStore.new
+      filters = opts.slice(:category, :store, :limit)
+      hits = products.search(query, **filters).map { |entry| entry.merge("product" => Index::EntryProduct.wire(entry)) }
+      { query: query, engine: products.search_engine, live: false, filters: filters, products: hits }
+    end
+    private_class_method :index_search
+
+    def self.format_index_search(result)
+      return "No index products match \"#{result[:query]}\"." if result[:products].empty?
+
+      lines = result[:products].map { |p| index_search_line(p) }
+      lines << "(From the local index, not live: check the price and stock with `portage find --store`.)"
+      lines << "(FTS5 isn't available in this SQLite, so this was a plain text match.)" if result[:engine] == "like"
+      lines.join("\n")
+    end
+    private_class_method :format_index_search
+
+    def self.index_search_line(product)
+      hosts = Array(product["stores"]).filter_map { |s| URI.parse(s["origin"].to_s).host }.join(", ")
+      ["#{product['title']}#{" — #{product['brand']}" if product['brand']}", ("(#{hosts})" unless hosts.empty?),
+       product["url"]].compact.join(" ")
+    end
+    private_class_method :index_search_line
+
     def self.run_index_add(argv)
-      json = argv.delete("--json") ? true : false
+      json = !argv.delete("--json").nil?
+      crawl = !argv.delete("--crawl").nil?
       url = argv.first && !argv.first.start_with?("-") ? argv.shift : nil
       unless url
         warn USAGE
         return 1
       end
 
-      result = Index::Builder.new.add(url)
+      result = Index::Builder.new.add(url, crawl: crawl)
       puts json ? JSON.pretty_generate(result) : result[:message]
       result[:added] ? 0 : 1
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: portage-cli
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Tom Whitbread
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-09-29 00:00:00.000000000 Z
+date: 2026-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: irb
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0.11'
 - !ruby/object:Gem::Dependency
   name: portage-ucp-client
   requirement: !ruby/object:Gem::Requirement
@@ -72,6 +72,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -155,6 +169,8 @@ files:
 - exe/portage
 - exe/portage-console
 - known-stores/categories.yml
+- known-stores/category-stoplist.yml
+- known-stores/category-synonyms.yml
 - lib/portage/cli.rb
 - lib/portage/cli/agent_profile_url.rb
 - lib/portage/cli/approval_policy.rb
@@ -184,8 +200,12 @@ files:
 - lib/portage/cli/buy.rb
 - lib/portage/cli/buyer_context.rb
 - lib/portage/cli/catalog_products.rb
+- lib/portage/cli/check.rb
+- lib/portage/cli/check_next_step.rb
 - lib/portage/cli/checkout_handoff.rb
 - lib/portage/cli/classifier.rb
+- lib/portage/cli/classifier/ranking.rb
+- lib/portage/cli/classifier/table.rb
 - lib/portage/cli/compare.rb
 - lib/portage/cli/confidence_check.rb
 - lib/portage/cli/config.rb
@@ -197,6 +217,7 @@ files:
 - lib/portage/cli/generate/adapter.rb
 - lib/portage/cli/generate/agent_profile.rb
 - lib/portage/cli/handoff_agents.rb
+- lib/portage/cli/handoff_host.rb
 - lib/portage/cli/handoff_only.rb
 - lib/portage/cli/handoff_reconciler.rb
 - lib/portage/cli/handoff_reconciler/wire_adapters.rb
@@ -209,12 +230,21 @@ files:
 - lib/portage/cli/human_prompt.rb
 - lib/portage/cli/index.rb
 - lib/portage/cli/index/builder.rb
+- lib/portage/cli/index/database.rb
+- lib/portage/cli/index/entry_product.rb
 - lib/portage/cli/index/exporter.rb
 - lib/portage/cli/index/known_cache.rb
+- lib/portage/cli/index/legacy_import.rb
 - lib/portage/cli/index/product_store.rb
+- lib/portage/cli/index/schema.rb
+- lib/portage/cli/index/search.rb
 - lib/portage/cli/index/sources.rb
 - lib/portage/cli/index/sources/browser.rb
 - lib/portage/cli/index/sources/shopify_catalog.rb
+- lib/portage/cli/index/sources/storefront_products.rb
+- lib/portage/cli/index/sources/storefront_products/mapper.rb
+- lib/portage/cli/index/sources/storefront_products/pages.rb
+- lib/portage/cli/index/sources/storefront_products/robots.rb
 - lib/portage/cli/index/sources/stores_file.rb
 - lib/portage/cli/index/sources/webmcp_sweep.rb
 - lib/portage/cli/index/sources/wikidata.rb