portage-cli 0.7.5 → 0.9.0

data/lib/portage/cli/index/builder.rb

@@ -0,0 +1,335 @@
+require "uri"
+require "portage/ucp/client"
+
+require_relative "../user_agent"
+require_relative "../classifier"
+require_relative "../probe_cache"
+require_relative "../handoff_only"
+require_relative "store"
+require_relative "product_store"
+require_relative "known_cache"
+require_relative "exporter"
+require_relative "sources"
+
+module Portage
+  module Cli
+    module Index
+      # `portage index build`/`refresh`/`add` — runs each configured
+      # Sources::* against the user's own machine, verifies any new origin
+      # with one `/.well-known/ucp` probe through the existing ProbeCache
+      # (throttled, capped at MAX_NEW_PROBES per run), and writes the
+      # survivors into Store/ProductStore.
+      #
+      # Never writes Portage::Ucp::Policy#merchant_allowlist, and never
+      # decides what `--yes` treats as "picked a store" — this only ever
+      # produces candidates SearchBackends::Index hands to `find`, on the
+      # same footing as any other search backend (see Store's own comment).
+      class Builder
+        CATALOG_CAP = "dev.ucp.shopping.catalog".freeze
+        CART_CAP = "dev.ucp.shopping.cart".freeze
+        CHECKOUT_CAP = "dev.ucp.shopping.checkout".freeze
+        CAPABILITY_PREFIXES = { "catalog" => CATALOG_CAP, "cart" => CART_CAP, "checkout" => CHECKOUT_CAP }.freeze
+
+        # Duplicated from Find::MAX_PROBES/THROTTLE rather than required —
+        # same reasoning as SearchBackends::Allowlist::TOTAL_CAP: this class
+        # is usable (and tested) with no dependency on find.rb.
+        MAX_NEW_PROBES = 500
+        THROTTLE = 0.1
+        STALE_AFTER = 7 * 24 * 60 * 60
+        TOP_CATEGORIES = 5
+
+        # Public so BrowserImport::Importer (Phase 3) labels a probed
+        # origin's capabilities exactly the way an index build does.
+        # @return [Array<String>] "catalog"/"cart"/"checkout", in that order.
+        def self.capabilities_of(session)
+          CAPABILITY_PREFIXES.filter_map do |label, prefix|
+            label if Array(session.capabilities).any? { |c| c == prefix || c.start_with?("#{prefix}.") }
+          end
+        end
+
+        def initialize(stores: Store.new, products: ProductStore.new, cache: ProbeCache.new, sources: nil,
+                       throttle: THROTTLE, max_new_probes: MAX_NEW_PROBES, out: $stdout, now: Time.now,
+                       known_cache: KnownCache.new, handoff_only: nil)
+          @stores = stores
+          @products = products
+          @cache = cache
+          @sources = sources || Sources.default
+          @throttle = throttle
+          @max_new_probes = max_new_probes
+          @out = out
+          @now = now
+          @known_cache = known_cache
+          @handoff_only = handoff_only || HandoffOnly.new
+        end
+
+        # @param queries [Array<String>, nil] passed through to every
+        #   source that takes one (today, just shopify_catalog).
+        # @param export [String, nil] a directory to also write a PR-ready
+        #   `{stores,products}.json` into (see Exporter) — nil (the
+        #   default) skips it.
+        # @return [Hash] a run summary — sources_run, candidates,
+        #   new_origins_checked, verified, products_added, capped, and
+        #   (with `export:`) exported.
+        def build(queries: nil, dry_run: false, export: nil)
+          sightings = gather(queries)
+          result = apply(sightings, dry_run: dry_run)
+          result[:exported] = Exporter.new(stores: @stores, products: @products).export(export) if export
+          result
+        end
+
+        # Re-verifies every entry older than 7 days (bumping last_verified
+        # only for the ones still answering), refreshes the known-stores
+        # cache unconditionally (Phase 2c — this is one of the three
+        # triggers the plan names, alongside a first-run fetch and a stale
+        # `doctor` check), then runs an ordinary #build, which itself adds
+        # anything new the sources turn up.
+        def refresh(queries: nil, dry_run: false, export: nil)
+          @known_cache.refresh! unless dry_run
+          reverify_stale(dry_run: dry_run)
+          build(queries: queries, dry_run: dry_run, export: export).merge(refreshed: true)
+        end
+
+        # `portage index add URL` — verifies and stores one origin
+        # directly, no source involved. A hand-off-only origin (Tier C,
+        # HandoffOnly) is recorded without ever probing it — the user
+        # explicitly named it, but that's still not a request this process
+        # sends.
+        def add(url)
+          origin = origin_of(url)
+          return { added: false, message: "Not a valid http(s) URL: #{url}" } unless origin
+          return store_manual_handoff_only(origin) if handoff_only_origin?(origin)
+
+          session = probe(origin)
+          store_manual(origin, session)
+        end
+
+        # `portage index remove HOST`
+        def remove(host)
+          removed = @stores.remove(host).positive?
+          { removed: removed,
+            message: removed ? "Removed #{host} from the index." : "No index entry for #{host}." }
+        end
+
+        private
+
+        def gather(queries)
+          @sources.flat_map { |source| candidates_from(source, queries) }
+        end
+
+        # One source failing shouldn't take the rest of the build with it —
+        # same posture as SearchBackends#urls_from/OfferSources.
+        def candidates_from(source, queries)
+          source.candidates(queries: queries).map { |c| c.merge(source: source.name) }
+        rescue StandardError
+          []
+        end
+
+        def apply(sightings, dry_run:)
+          grouped = sightings.group_by { |s| s[:origin] }
+          probed = probe_new_origins(grouped, dry_run: dry_run)
+          products_added = dry_run ? 0 : store_products(sightings)
+          { sources_run: @sources.map(&:name), candidates: sightings.length, products_added: products_added,
+            new_origins_checked: probed[:checked], verified: probed[:verified], capped: probed[:capped] }
+        end
+
+        def probe_new_origins(grouped, dry_run:)
+          state = { checked: [], verified: [], probes: 0, capped: false }
+          grouped.each { |origin, group| probe_one_new_origin(origin, group, state, dry_run: dry_run) }
+          state.slice(:checked, :verified, :capped)
+        end
+
+        def probe_one_new_origin(origin, group, state, dry_run:)
+          if @stores.find(origin)
+            update_existing(origin, group) unless dry_run
+            return
+          end
+          return handoff_only_new_origin(origin, group, dry_run: dry_run) if handoff_only_origin?(origin)
+          return state[:capped] = true if state[:probes] >= @max_new_probes
+
+          throttle(state[:probes])
+          state[:probes] += 1
+          state[:checked] << origin
+          progress("Probing #{origin}...")
+          session = probe(origin)
+          return unless session
+
+          state[:verified] << origin
+          store_new(origin, session, group) unless dry_run
+        end
+
+        # Never spends a probe (docs/plans/buy-skill-and-local-browser.md
+        # Phase 5) — recorded straight as `handoff_only: true`, same as a
+        # `store_new` verdict but with no capabilities and no request ever
+        # made.
+        def handoff_only_new_origin(origin, group, dry_run:)
+          return if dry_run
+
+          @stores.upsert(origin, platform: platform_of(group), capabilities: [],
+                                 categories: merge_categories({}, group), sources: merged_sources(nil, group),
+                                 last_verified: @now.to_i, handoff_only: true)
+        end
+
+        def handoff_only_origin?(origin) = @handoff_only.host?(host_of(origin))
+
+        def host_of(origin)
+          URI.parse(origin).host
+        rescue URI::InvalidURIError
+          nil
+        end
+
+        def store_manual_handoff_only(origin)
+          existing = @stores.find(origin)
+          sources = ((existing && existing["sources"]) || []) + ["manual"]
+          @stores.upsert(origin, sources: sources.uniq, last_verified: @now.to_i, handoff_only: true)
+          { added: true, origin: origin, message: "Added #{origin} — hand-off only, never probed." }
+        end
+
+        def update_existing(origin, group)
+          existing = @stores.find(origin)
+          @stores.upsert(origin, sources: merged_sources(existing, group),
+                                 categories: merge_categories(existing["categories"], group))
+        end
+
+        def store_new(origin, session, group)
+          @stores.upsert(origin, platform: platform_of(group), capabilities: capabilities_of(session),
+                                 categories: merge_categories({}, group), sources: merged_sources(nil, group),
+                                 last_verified: @now.to_i, handoff_only: false)
+        end
+
+        def store_manual(origin, session)
+          existing = @stores.find(origin)
+          sources = ((existing && existing["sources"]) || []) + ["manual"]
+          if session
+            @stores.upsert(origin, platform: existing && existing["platform"], capabilities: capabilities_of(session),
+                                   sources: sources.uniq, last_verified: @now.to_i, handoff_only: false)
+            { added: true, origin: origin, message: "Added #{origin} — verified UCP." }
+          else
+            @stores.upsert(origin, sources: sources.uniq, last_verified: @now.to_i, handoff_only: true)
+            { added: true, origin: origin,
+              message: "Added #{origin} — no /.well-known/ucp response; marked hand-off only." }
+          end
+        end
+
+        def merged_sources(existing, group)
+          (Array(existing && existing["sources"]) + group.map { |g| g[:source] }).uniq
+        end
+
+        def merge_categories(existing, group)
+          tally = Hash.new(0)
+          Array(existing).each { |id, weight| tally[id] += weight.to_i }
+          group.each do |sighting|
+            next unless sighting[:title]
+
+            Classifier.categories_for(sighting[:title]).each { |id| tally[id] += 1 }
+          end
+          tally.sort_by { |_id, weight| -weight }.first(TOP_CATEGORIES).to_h
+        end
+
+        def platform_of(group)
+          "shopify" if group.any? { |g| g[:source] == "shopify_catalog" }
+        end
+
+        def capabilities_of(session) = self.class.capabilities_of(session)
+
+        def store_products(sightings)
+          eligible = sightings.select { |s| s[:title] && @stores.find(s[:origin]) }
+          eligible.each { |sighting| store_product(sighting) }
+          eligible.length
+        end
+
+        def store_product(sighting)
+          key = product_key(sighting)
+          category = Classifier.categories_for(sighting[:title]).first
+          @products.upsert(key, origin: sighting[:origin], seen_at: @now.to_i, title: sighting[:title],
+                                brand: sighting[:brand], gtin: sighting[:gtin], category: category,
+                                sources: [sighting[:source]].compact)
+        end
+
+        # GTIN when a source has one (none do yet); otherwise a normalized
+        # brand+title slug, stable across runs so the same product seen
+        # again updates its entry instead of duplicating it.
+        def product_key(sighting)
+          return "gtin:#{sighting[:gtin]}" if sighting[:gtin]
+
+          slug = [sighting[:brand], sighting[:title]].compact.join(" ").downcase.gsub(/[^a-z0-9]+/, "-")
+          "title:#{slug}"
+        end
+
+        # Skips by whether the origin is *currently* hand-off only
+        # (`handoff_only_origin?`, checked against live config), never by
+        # the flag a previous run happened to store — those disagree in
+        # both directions: an origin whose UCP probe simply failed is
+        # stored `handoff_only: true` too (#store_new/#store_manual) but
+        # isn't on the Tier C list and should keep getting re-verified in
+        # case it comes online, while a stale entry recorded *before* the
+        # user added its host to `handoff_only_hosts` is stored
+        # `handoff_only: false` and must stop being probed the moment that
+        # config changes, without waiting for some other source to
+        # re-sight it. An origin caught by the live check gets its stored
+        # flag flipped to match — no request, just a `last_verified` bump
+        # so it isn't re-checked again until the next stale window.
+        def reverify_stale(dry_run:)
+          probes = 0
+          @stores.all.select { |e| stale?(e) }.each do |entry|
+            origin = entry["origin"]
+            next reverify_now_handoff_only(origin, dry_run: dry_run) if handoff_only_origin?(origin)
+            break if probes >= @max_new_probes
+
+            throttle(probes)
+            probes += 1
+            progress("Re-verifying #{origin}...")
+            session = probe(origin)
+            @stores.upsert(origin, capabilities: capabilities_of(session), last_verified: @now.to_i) \
+              if session && !dry_run
+          end
+        end
+
+        def reverify_now_handoff_only(origin, dry_run:)
+          return if dry_run
+
+          @stores.upsert(origin, handoff_only: true, last_verified: @now.to_i)
+        end
+
+        def stale?(entry)
+          verified = entry["last_verified"]
+          verified.nil? || (@now.to_i - verified.to_i) > STALE_AFTER
+        end
+
+        # A cached *miss* saves the work; a cached hit still has to connect
+        # for the manifest details this run wants (same posture as
+        # Find#probe).
+        def probe(origin)
+          return nil if @cache.fetch(origin) == false
+
+          session = discover(origin)
+          @cache.record(origin, !session.nil?)
+          session
+        end
+
+        def discover(origin)
+          Portage::Ucp::Client.discover(origin, headers: UserAgent.headers)
+        rescue StandardError
+          nil
+        end
+
+        def throttle(probed)
+          sleep(@throttle) if probed.positive? && @throttle.to_f.positive?
+        end
+
+        def progress(message)
+          @out&.puts(message)
+        end
+
+        def origin_of(url)
+          uri = URI.parse(url.to_s)
+          return nil unless uri.host && uri.scheme.to_s.start_with?("http")
+
+          port = uri.port == uri.default_port ? "" : ":#{uri.port}"
+          "#{uri.scheme}://#{uri.host}#{port}"
+        rescue URI::InvalidURIError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/index/exporter.rb

@@ -0,0 +1,91 @@
+require "json"
+require "fileutils"
+
+module Portage
+  module Cli
+    module Index
+      # `portage index build --export DIR` — writes a PR-ready copy of the
+      # user's own local index into `DIR/{stores,products}.json`, in the
+      # same shape as `portage-cli/known-stores/` in the repo, so a
+      # maintainer's run is just "run this, `git add`, open a PR" (Decision
+      # 1a: the user builds the index, not CI).
+      #
+      # Nothing personal ships in the export. PERSONAL_SOURCES are the
+      # labels a browser import writes (`history`/`bookmark` — Phase 3 —
+      # plus Phase 2b's placeholder `browser`): a store entry whose only
+      # `sources` are personal is dropped outright, and they're stripped
+      # from the `sources` of any entry that also has a real source, so a
+      # store found both by `shopify_catalog` and by browsing history
+      # exports as if browsing history had never touched it. A product
+      # entry is filtered the same way on its own `sources` (Phase 3 adds
+      # that field — so a product page kept by `browser import
+      # --include-product-pages` never exports, even at a store a real
+      # source also found), and in any case is kept only if at least one
+      # of its `stores[].origin` survived the store filter, with its
+      # `stores` array trimmed to just those surviving origins (entries
+      # written before Phase 3 carry no `sources`, so that origin check is
+      # all they get).
+      class Exporter
+        PERSONAL_SOURCES = %w[browser history bookmark].freeze
+
+        def initialize(stores:, products:)
+          @stores = stores
+          @products = products
+        end
+
+        # @return [Hash] stores:, products: — counts written, plus dir.
+        def export(dir)
+          FileUtils.mkdir_p(dir)
+          exportable_stores = @stores.all.filter_map { |entry| exportable_store(entry) }
+          exportable_origins = exportable_stores.map { |e| e["origin"] }
+          exportable_products = @products.all.filter_map { |entry| exportable_product(entry, exportable_origins) }
+
+          write(dir, "stores.json", exportable_stores, key: "origin")
+          write(dir, "products.json", exportable_products, key: "key")
+
+          { dir: dir, stores: exportable_stores.length, products: exportable_products.length }
+        end
+
+        private
+
+        # nil (never exported) when browsing history/bookmarks are the
+        # *only* source; otherwise the entry with every personal label
+        # stripped out of its own `sources` list, so a mixed-source entry
+        # still exports — just with no trace that browsing history ever
+        # found it too.
+        def exportable_store(entry)
+          return nil if personal_only?(entry)
+
+          entry.merge("sources" => Array(entry["sources"]) - PERSONAL_SOURCES)
+        end
+
+        # A product with no surviving store origin is exactly as personal
+        # as a store entry with only "browser" as its source — it only
+        # exists in the index because of something the user looked at, not
+        # something a source discovered — so it's dropped the same way.
+        def exportable_product(entry, exportable_origins)
+          return nil if personal_only?(entry)
+
+          kept_stores = Array(entry["stores"]).select { |s| exportable_origins.include?(s["origin"]) }
+          return nil if kept_stores.empty?
+
+          exported = entry.merge("stores" => kept_stores)
+          entry.key?("sources") ? exported.merge("sources" => Array(entry["sources"]) - PERSONAL_SOURCES) : exported
+        end
+
+        # True when the entry names at least one source and every one of
+        # them is personal. An entry with no `sources` (a product written
+        # before Phase 3) or an empty list isn't judged here at all.
+        def personal_only?(entry)
+          sources = Array(entry["sources"])
+          !sources.empty? && (sources - PERSONAL_SOURCES).empty?
+        end
+
+        def write(dir, filename, entries, key:)
+          data = entries.to_h { |entry| [entry[key], entry] }
+          File.write(File.join(dir, filename), JSON.generate(data))
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/index/known_cache.rb

@@ -0,0 +1,155 @@
+require "net/http"
+require "uri"
+require "json"
+require "fileutils"
+require "timeout"
+require "portage/ucp/support/connection"
+
+require_relative "../known_stores_url"
+require_relative "../user_agent"
+
+module Portage
+  module Cli
+    module Index
+      # The repo's own `known-stores/{stores,products}.json`, fetched over
+      # jsdelivr `@main` and cached at `~/.portage/index/known-{stores,
+      # products}.json` (docs/plans/buy-skill-and-local-browser.md Phase
+      # 2c). Same untrusted-data posture as Store/ProductStore — see their
+      # own comments at the top of this directory — plus one more rule:
+      # the user's own entries always win, so SearchBackends::Index merges
+      # this cache *underneath* Store/ProductStore rather than the other
+      # way round; this class never writes into Store/ProductStore itself.
+      #
+      # Fetched lazily the first time a caller needs it and no cache file
+      # exists yet (SearchBackends::Index#available?/#search), refreshed
+      # unconditionally by `portage index refresh`, and refreshed by
+      # `doctor` when it finds the cache older than STALE_AFTER. A fetch
+      # failure — offline, timeout, bad JSON, a non-2xx response — is
+      # swallowed exactly like SearchBackends/OfferSources::ShopifyCatalog:
+      # a stale or missing cache just means `find` works the way it did
+      # before this phase.
+      class KnownCache
+        STORES_PATH = File.join(Dir.home, ".portage", "index", "known-stores.json").freeze
+        PRODUCTS_PATH = File.join(Dir.home, ".portage", "index", "known-products.json").freeze
+
+        STALE_AFTER = 7 * 24 * 60 * 60
+        TIMEOUT = 5
+
+        # No known-stores/products entry may carry a price or stock field
+        # — Store/ProductStore's own comment says those are always live,
+        # and a fetched file is one more untrusted input than a locally
+        # built one, so this is checked again on the way in rather than
+        # trusting the source repo's own review (there's no CI gate for
+        # it — Decision 1a, "no central artifact, so no signing").
+        FORBIDDEN_FIELDS = %w[price amount stock].freeze
+
+        def initialize(stores_path: STORES_PATH, products_path: PRODUCTS_PATH,
+                       stores_url: KnownStoresUrl::STORES, products_url: KnownStoresUrl::PRODUCTS)
+          @stores_path = stores_path
+          @products_path = products_path
+          @stores_url = stores_url
+          @products_url = products_url
+        end
+
+        # @return [Hash] origin => entry, same schema as Index::Store's own
+        #   file.
+        def stores = read(@stores_path)
+
+        # @return [Hash] key => entry, same schema as Index::ProductStore's
+        #   own file.
+        def products = read(@products_path)
+
+        def exists? = File.exist?(@stores_path) || File.exist?(@products_path)
+
+        # @return [Integer, nil] seconds since the stores cache was last
+        #   fetched — its own file mtime, since the cache is written in the
+        #   same schema as the local index and carries no fetch timestamp
+        #   of its own. nil when there's no cache yet.
+        def age(now: Time.now)
+          return nil unless File.exist?(@stores_path)
+
+          now.to_i - File.mtime(@stores_path).to_i
+        end
+
+        def stale?(now: Time.now)
+          a = age(now: now)
+          a.nil? || a > STALE_AFTER
+        end
+
+        # The "first run" trigger: only fetches when there's no cache file
+        # at all — a stale-but-present cache is left for `index
+        # refresh`/`doctor` to decide about, not silently re-fetched on
+        # every `find`.
+        # @return [Boolean] whether a fetch happened (and its outcome).
+        def fetch_if_missing!
+          return false if exists?
+
+          refresh!
+        end
+
+        # `index refresh` and a stale `doctor` check call this
+        # unconditionally.
+        # @return [Boolean] true when at least one of the two files
+        #   fetched successfully.
+        def refresh!
+          stores_ok = fetch_and_cache(@stores_url, @stores_path)
+          products_ok = fetch_and_cache(@products_url, @products_path)
+          stores_ok || products_ok
+        end
+
+        private
+
+        def fetch_and_cache(url, path)
+          data = Timeout.timeout(TIMEOUT) { fetch(url) }
+          return false unless valid?(data)
+
+          write(path, sanitize(data))
+          true
+        rescue StandardError
+          false
+        end
+
+        def fetch(url)
+          uri = URI.parse(url)
+          response = Portage::Ucp::Support::Connection.start(
+            uri, route: :search, open_timeout: TIMEOUT, read_timeout: TIMEOUT
+          ) { |http| http.get(uri.request_uri, UserAgent.headers) }
+          return nil unless response.is_a?(Net::HTTPSuccess)
+
+          JSON.parse(response.body)
+        end
+
+        # Same shape as Store/ProductStore's own file: a Hash keyed by
+        # origin (stores) or product key (products), each value itself a
+        # Hash. Anything else — an array, a string, a top-level list — is a
+        # fetch gone wrong, not a cache worth keeping.
+        def valid?(data)
+          data.is_a?(Hash) && data.values.all?(Hash)
+        end
+
+        def sanitize(data)
+          data.transform_values { |entry| entry.reject { |k, _| FORBIDDEN_FIELDS.include?(k.to_s) } }
+        end
+
+        # A cache that can't be written just means this fetch's findings
+        # aren't saved — never a crash (same posture as Store/
+        # ProductStore#write).
+        def write(path, data)
+          FileUtils.mkdir_p(File.dirname(path))
+          File.write(path, JSON.generate(data))
+        rescue StandardError
+          nil
+        end
+
+        def read(path)
+          return {} unless File.readable?(path)
+
+          parsed = JSON.parse(File.read(path, encoding: "UTF-8"))
+          parsed.is_a?(Hash) ? parsed : {}
+        rescue StandardError
+          {}
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/index/product_store.rb

@@ -0,0 +1,101 @@
+require "json"
+require "fileutils"
+
+module Portage
+  module Cli
+    module Index
+      # `~/.portage/index/products.json` — product identities the index has
+      # seen, one entry per GTIN/MPN (when a source gives one) or per
+      # title+brand otherwise. **No prices or stock** — those are always
+      # live, read straight from the store's own catalog at buy time
+      # (docs/plans/buy-skill-and-local-browser.md Phase 2b).
+      #
+      # Entry shape: title, brand, gtin, category, aliases (other titles the
+      # same product went by at a different store), stores (array of
+      # {origin:, last_seen:}), sources (which index sources ever saw it —
+      # Phase 3 adds this so Index::Exporter can tell a product only a
+      # browser import saw from one a real source found; entries written
+      # before it have none).
+      class ProductStore
+        PATH = File.join(Dir.home, ".portage", "index", "products.json").freeze
+
+        def initialize(path: PATH)
+          @path = path
+        end
+
+        def all = entries.values
+
+        def find(key) = entries[key]
+
+        # @param key [String] a stable key for this product — the source's
+        #   GTIN/MPN when it has one, else a normalized title+brand.
+        # @param origin [String] the store this sighting came from.
+        # @param seen_at [Integer] unix seconds.
+        def upsert(key, origin:, seen_at:, **fields)
+          existing = entries[key] || { "key" => key, "aliases" => [], "stores" => [] }
+          aliases = merge_aliases(existing, fields[:title])
+          merged = existing.merge(fields.transform_keys(&:to_s)) { |field, old, new| merge_field(field, old, new) }
+          merged["aliases"] = aliases
+          merged["stores"] = merge_stores(merged["stores"], origin, seen_at)
+          entries[key] = merged
+          write
+          merged
+        end
+
+        def exists? = File.exist?(@path)
+
+        private
+
+        # `aliases`/`stores` accumulate across upserts (merged separately
+        # below), `sources` is the union of every sighting's; every other
+        # field (title, brand, gtin, category) is just the latest
+        # sighting's value, since a later source's read is no less
+        # authoritative than an earlier one's.
+        def merge_field(field, old, new)
+          return (Array(old) + Array(new)).uniq if field == "sources"
+
+          %w[aliases stores].include?(field) ? old : (new || old)
+        end
+
+        # The *previous* title becomes an alias when a later sighting gives
+        # a different one — the newest sighting's title stays canonical
+        # (see #merge_field), and the one it replaced is kept so a search
+        # for either still finds this product.
+        def merge_aliases(existing, new_title)
+          old_title = existing["title"]
+          return Array(existing["aliases"]) if new_title.nil? || old_title.nil? || new_title == old_title
+
+          (Array(existing["aliases"]) + [old_title]).uniq
+        end
+
+        def merge_stores(stores, origin, seen_at)
+          kept = Array(stores).reject { |s| s["origin"] == origin }
+          (kept + [{ "origin" => origin, "last_seen" => seen_at }]).sort_by { |s| s["origin"] }
+        end
+
+        def entries
+          @entries ||= read
+        end
+
+        def read
+          return {} unless File.readable?(@path)
+
+          # See Index::Store#read's comment — a product title routinely
+          # carries non-ASCII bytes, so this has to read as UTF-8 rather
+          # than whatever the process's default external encoding is.
+          parsed = JSON.parse(File.read(@path, encoding: "UTF-8"))
+          parsed.is_a?(Hash) ? parsed : {}
+        rescue StandardError
+          {}
+        end
+
+        def write
+          FileUtils.mkdir_p(File.dirname(@path))
+          File.write(@path, JSON.generate(@entries))
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end