portage-cli 0.10.0 → 0.12.0

data/lib/portage/cli/index/sources/storefront_products/pages.rb

@@ -0,0 +1,114 @@
+require "net/http"
+require "uri"
+require "json"
+require "portage/ucp/support/connection"
+
+require_relative "../../../user_agent"
+
+module Portage
+  module Cli
+    module Index
+      module Sources
+        class StorefrontProducts
+          # The HTTP half of a crawl: GETs `products.json?limit=&page=`
+          # through Support::Connection with the shared UserAgent, pausing
+          # between pages, waiting out one 429's Retry-After and stopping on
+          # a second, and turning anything that isn't a products array into
+          # a reason.
+          class Pages
+            PAUSE = 1
+            MAX_RETRY_AFTER = 60
+            DEFAULT_RETRY_AFTER = 10
+            OPEN_TIMEOUT = 5
+            READ_TIMEOUT = 20
+
+            def self.get(url, accept: "application/json")
+              uri = URI.parse(url)
+              headers = UserAgent.headers.merge("Accept" => accept)
+              Portage::Ucp::Support::Connection.start(uri, route: :store, open_timeout: OPEN_TIMEOUT,
+                                                           read_timeout: READ_TIMEOUT) do |http|
+                http.get(uri.request_uri, headers)
+              end
+            end
+
+            # @param allowed [#call] path -> whether robots.txt allows it.
+            def initialize(endpoint, per_page:, max_pages:, sleeper:, allowed: ->(_path) { true })
+              @endpoint = endpoint
+              @path = URI.parse(endpoint).path
+              @allowed = allowed
+              @per_page = per_page
+              @max_pages = max_pages
+              @sleeper = sleeper
+              @rate_limited = 0
+            end
+
+            # Yields each non-empty page's raw products.
+            # @return [Array(String, String, Integer)] status ("ok",
+            #   "partial", "skipped"), reason (nil when ok) and pages read.
+            def each(&)
+              (1..@max_pages).each do |page|
+                @sleeper.call(PAUSE) if page > 1
+                products = fetch(page)
+                done = outcome(products, page, &)
+                return done if done
+              end
+              ["partial", "page_cap", @max_pages]
+            end
+
+            private
+
+            # nil means "full page, keep going".
+            def outcome(products, page)
+              return failed(products, page - 1) unless products.is_a?(Array)
+              return ["skipped", "empty", 0] if products.empty? && page == 1
+
+              yield products unless products.empty?
+              ["ok", nil, page] if products.length < @per_page
+            end
+
+            # Past page 1 a failure keeps what earlier pages found.
+            def failed(reason, pages) = [pages.zero? ? "skipped" : "partial", reason, pages]
+
+            # @return [Array<Hash>, String] a page's products, or why there
+            #   are none.
+            def fetch(page)
+              return "robots" unless @allowed.call("#{@path}#{query(page)}")
+
+              response = self.class.get("#{@endpoint}#{query(page)}")
+              return products_in(response) unless response.code == "429"
+
+              @rate_limited += 1
+              return "rate_limited" if @rate_limited > 1
+
+              @sleeper.call(retry_after(response["Retry-After"]))
+              fetch(page)
+            rescue StandardError => e
+              "error: #{e.class}"
+            end
+
+            # Page 1 is the bare URL (Shopify's default page), so a robots
+            # rule aimed at duplicate `?page=1` URLs doesn't read as a ban
+            # on the whole catalogue.
+            def query(page) = page == 1 ? "?limit=#{@per_page}" : "?limit=#{@per_page}&page=#{page}"
+
+            def products_in(response)
+              return "not_found" if response.code == "404"
+              return "redirect" if response.is_a?(Net::HTTPRedirection)
+              return "http_#{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+              body = JSON.parse(response.body.to_s.dup.force_encoding(Encoding::UTF_8))
+              body.is_a?(Hash) && body["products"].is_a?(Array) ? body["products"] : "not_json"
+            rescue JSON::ParserError
+              "not_json"
+            end
+
+            def retry_after(value)
+              seconds = Integer(value.to_s, exception: false) || DEFAULT_RETRY_AFTER
+              seconds.clamp(0, MAX_RETRY_AFTER)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/index/sources/storefront_products/robots.rb

@@ -0,0 +1,70 @@
+module Portage
+  module Cli
+    module Index
+      module Sources
+        class StorefrontProducts
+          # Just enough robots.txt (RFC 9309) to answer "may this agent GET
+          # this path?": the group naming this agent if there is one, else
+          # the `*` group; `*` and `$` in rules; the longest matching rule
+          # wins, and Allow wins a tie. No rules, or no file, means allowed.
+          class Robots
+            def initialize(body)
+              @groups = parse(body.to_s)
+            end
+
+            # @param path [String] request path, query included.
+            # @param agent [String] the User-Agent this process sends.
+            def allowed?(path, agent:)
+              rules = rules_for(agent.to_s.downcase)
+              matches = rules.select { |_allow, pattern| match?(pattern, path) }
+              return true if matches.empty?
+
+              longest = matches.map { |_allow, pattern| pattern.length }.max
+              matches.select { |_allow, pattern| pattern.length == longest }.any?(&:first)
+            end
+
+            private
+
+            def rules_for(agent)
+              named = @groups.select { |agents, *| agents.any? { |a| a != "*" && agent.include?(a) } }
+              chosen = named.empty? ? @groups.select { |agents, *| agents.include?("*") } : named
+              chosen.flat_map { |_agents, rules, _closed| rules }
+            end
+
+            # @return [Array<Array>] [agents, [[allow?, pattern], ...], closed]
+            #   per group.
+            def parse(body)
+              groups = []
+              body.each_line do |line|
+                field, value = line.sub(/#.*/, "").split(":", 2).map { |part| part.to_s.strip }
+                next if value.nil?
+
+                add_line(groups, field.downcase, value)
+              end
+              groups
+            end
+
+            # A rule line (even an empty `Disallow:`) ends a group's
+            # User-agent list, so the next User-agent starts a new group.
+            def add_line(groups, field, value)
+              if field == "user-agent"
+                groups << [[], [], false] if groups.empty? || groups.last[2]
+                groups.last[0] << value.downcase
+              elsif %w[allow disallow].include?(field) && groups.any?
+                groups.last[2] = true
+                groups.last[1] << [field == "allow", value] unless value.empty?
+              end
+            end
+
+            def match?(pattern, path)
+              anchored = pattern.end_with?("$")
+              body = anchored ? pattern.chomp("$") : pattern
+              regex = body.split("*", -1).map { |part| Regexp.escape(part) }.join(".*")
+              Regexp.new("\\A#{regex}#{'\\z' if anchored}").match?(path)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/index/sources/storefront_products.rb

@@ -0,0 +1,147 @@
+require "net/http"
+require "uri"
+
+require_relative "../../user_agent"
+require_relative "../../classifier"
+require_relative "../../handoff_only"
+require_relative "../store"
+require_relative "storefront_products/mapper"
+require_relative "storefront_products/robots"
+require_relative "storefront_products/pages"
+
+module Portage
+  module Cli
+    module Index
+      module Sources
+        # A store's own public catalogue, read from Shopify's `/products.json`
+        # (docs/plans/local-catalogue.md Phase 2), for origins already in
+        # the index. Off by default: it runs only from `portage index build
+        # --sources storefront_products` or `portage index add URL --crawl`,
+        # never from `find` or `buy`.
+        #
+        # Guardrails: a hand-off-only host (HandoffOnly, checked live) and a
+        # robots.txt Disallow get no products.json request at all; pages are
+        # MAX_PAGES x 250 at most, PAUSE seconds apart; a 429 waits out its
+        # Retry-After once and a second 429 stops that store; a 404, a
+        # redirect, an empty page 1, or anything that isn't products JSON
+        # (a bot wall) skips the store. Every outcome is noted on the store
+        # row as `crawl` ({status, reason, pages, products, at}).
+        #
+        # Yields Mapper sightings (no price, no availability) plus one store
+        # sighting per origin carrying `store_fields:` (the crawl note, and
+        # platform "shopify" when products.json answered); Index::Builder
+        # writes both.
+        class StorefrontProducts
+          MAX_PAGES = 20
+          MAX_STORES = 25
+          PER_PAGE = 250
+
+          # The one platform seam: WooCommerce's Store API can join here.
+          # nil platform means "not known yet", worth a try.
+          def self.endpoint_for(origin, platform)
+            "#{origin}/products.json" if platform.nil? || platform == "shopify"
+          end
+
+          # The guardrails, plus test seams for sleeping and the clock.
+          def initialize(stores: Store.new, handoff_only: HandoffOnly.new, max_pages: MAX_PAGES,
+                         max_stores: MAX_STORES, per_page: PER_PAGE, sleeper: Kernel.method(:sleep), now: Time.now)
+            @stores = stores
+            @handoff_only = handoff_only
+            @max_pages = max_pages
+            @max_stores = max_stores
+            @per_page = per_page
+            @sleeper = sleeper
+            @now = now
+          end
+
+          def name = "storefront_products"
+
+          def description
+            "Each indexed store's own /products.json (Shopify), up to #{MAX_PAGES} pages a store and " \
+              "#{MAX_STORES} stores a run, least recently crawled first. Off by default — opt in with " \
+              "--sources storefront_products or `index add URL --crawl`."
+          end
+
+          def source_path = nil
+
+          # `**` accepts (and ignores) the shared Source#candidates(queries:)
+          # interface.
+          def candidates(**)
+            eligible = @stores.all.select do |entry|
+              self.class.endpoint_for(entry["origin"], entry["platform"]) && !handoff_only?(entry["origin"])
+            end
+            eligible.sort_by { |entry| entry.dig("crawl", "at").to_i }.first(@max_stores)
+                    .flat_map { |entry| crawl(entry["origin"], platform: entry["platform"]) }
+          end
+
+          # @return [Array<Hash>] product sightings, then the store sighting.
+          def crawl(origin, platform: nil)
+            endpoint = self.class.endpoint_for(origin, platform)
+            robots = robots_for(origin, endpoint)
+            return [store_sighting(origin, note("skipped", robots, 0, 0))] if robots.is_a?(String)
+
+            sightings = []
+            classify = memoized_classifier
+            status, reason, pages = pages_for(endpoint, robots).each do |products|
+              sightings.concat(products.map { |raw| Mapper.sighting(raw, origin: origin, classify: classify) })
+            end
+            sightings + [store_sighting(origin, note(status, reason, pages, sightings.length))]
+          rescue StandardError => e
+            [store_sighting(origin, note("skipped", "error: #{e.class}", 0, 0))]
+          end
+
+          private
+
+          def pages_for(endpoint, robots)
+            Pages.new(endpoint, per_page: @per_page, max_pages: @max_pages, sleeper: @sleeper,
+                                allowed: ->(path) { robots.allowed?(path, agent: UserAgent.value) })
+          end
+
+          # The store's robots.txt rules, or why the crawl stops before
+          # any request (no endpoint, hand-off only) or before any
+          # products.json request (robots.txt unreachable). Pages checks
+          # the rules against every page URL it would request.
+          # RFC 9309: a 4xx robots.txt means no rules; a 5xx or no answer
+          # at all means stay out.
+          # @return [Robots, String]
+          def robots_for(origin, endpoint)
+            return "unsupported_platform" unless endpoint
+            return "handoff_only" if handoff_only?(origin)
+
+            response = Pages.get("#{origin}/robots.txt", accept: "text/plain")
+            return Robots.new(nil) if response.code.start_with?("4")
+            return "redirect" if response.is_a?(Net::HTTPRedirection)
+            return "robots_unreachable" unless response.is_a?(Net::HTTPSuccess)
+
+            Robots.new(response.body.to_s.dup.force_encoding(Encoding::UTF_8))
+          rescue StandardError
+            "robots_unreachable"
+          end
+
+          # Product types and tags repeat across a catalogue, and Classifier
+          # re-reads its YAML on every call.
+          def memoized_classifier
+            memo = {}
+            ->(text) { memo[text] ||= Classifier.categories_for(text) }
+          end
+
+          def handoff_only?(origin)
+            @handoff_only.host?(URI.parse(origin.to_s).host)
+          rescue URI::InvalidURIError
+            true
+          end
+
+          def note(status, reason, pages, products)
+            { "status" => status, "reason" => reason, "pages" => pages, "products" => products, "at" => @now.to_i }
+          end
+
+          def store_sighting(origin, crawl)
+            fields = { crawl: crawl }
+            fields[:platform] = "shopify" unless crawl["status"] == "skipped"
+            { origin: origin, url: nil, title: nil, brand: nil, gtin: nil, store_fields: fields }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/index/sources.rb

@@ -3,6 +3,7 @@ require_relative "sources/stores_file"
 require_relative "sources/browser"
 require_relative "sources/wikidata"
 require_relative "sources/webmcp_sweep"
+require_relative "sources/storefront_products"
 
 module Portage
   module Cli
@@ -16,7 +17,8 @@ module Portage
           "stores_file" => -> { StoresFile.new },
           "browser" => -> { Browser.new },
           "wikidata" => -> { Wikidata.new },
-          "webmcp_sweep" => -> { WebmcpSweep.new }
+          "webmcp_sweep" => -> { WebmcpSweep.new },
+          "storefront_products" => -> { StorefrontProducts.new }
         }.freeze
 
         # Runs with no extra opt-in: no bridge required, and no low-yield
@@ -24,7 +26,8 @@ module Portage
         # show up in `portage index sources`, just not run unless named in
         # `--sources`. `browser` is listed but never a default — it yields
         # nothing; `portage browser import` writes those entries itself
-        # (see Sources::Browser).
+        # (see Sources::Browser). `storefront_products` sends up to 21
+        # requests a store, so it runs only when named.
         DEFAULT_NAMES = %w[shopify_catalog stores_file].freeze
 
         def self.all = ALL.values.map(&:call)

data/lib/portage/cli/index/store.rb

@@ -1,12 +1,13 @@
-require "json"
-require "fileutils"
 require "uri"
+require_relative "database"
 
 module Portage
   module Cli
     module Index
-      # `~/.portage/index/stores.json` — one entry per merchant origin
-      # `portage index build`/`refresh`/`add` has found and verified.
+      # The `stores` table of `~/.portage/index/index.sqlite3` (Index::Database;
+      # `stores.json` before docs/plans/local-catalogue.md Phase 1) — one
+      # entry per merchant origin `portage index build`/`refresh`/`add` has
+      # found and verified.
       #
       # This is untrusted, user-owned data, same posture as `stores.yml`
       # before it's tagged: never checked into git (docs/plans/
@@ -19,7 +20,7 @@ module Portage
       # search result (see search_backends_spec.rb/cli_spec.rb for the
       # specs proving both).
       #
-      # Entry shape (string keys, JSON on disk): origin, platform,
+      # Entry shape (string keys, a JSON object per row): origin, platform,
       # ucp_version, capabilities (array of "catalog"/"cart"/"checkout"),
       # webmcp_preset, categories (category id => weight, top 5),
       # sources (array of source names that found it), last_verified
@@ -27,45 +28,49 @@ module Portage
       class Store
         PATH = File.join(Dir.home, ".portage", "index", "stores.json").freeze
 
+        # @param path [String] where the legacy stores.json lives (or would
+        #   live) — the database sits beside it, and a stores.json found
+        #   there is imported on first open.
         def initialize(path: PATH)
-          @path = path
+          @db = Database.new(path: Database.path_for(path))
         end
 
-        def all = entries.values
+        def all = @db.entries("stores").values
 
-        def find(origin) = entries[origin]
+        def find(origin) = @db.get("stores", origin)
 
         # Merges `fields` onto whatever's already there for `origin` (or
         # starts a fresh entry) — so a second source finding the same store
         # adds to its `sources`/`categories` rather than clobbering the
         # first source's findings.
         def upsert(origin, **fields)
-          existing = entries[origin] || { "origin" => origin }
-          entries[origin] = existing.merge(fields.transform_keys(&:to_s))
-          write
-          entries[origin]
+          @db.transaction do
+            existing = @db.get("stores", origin) || { "origin" => origin }
+            existing.merge(fields.transform_keys(&:to_s)).tap { |merged| @db.put("stores", origin, merged) }
+          end
         end
 
         # @return [Integer] how many entries (0 or 1 — origins are unique)
         #   this host's removal actually dropped.
         def remove(host)
-          before = entries.length
-          entries.reject! { |origin, _| host_of(origin) == host }
-          write
-          before - entries.length
+          @db.transaction do
+            doomed = @db.entries("stores").keys.select { |origin| host_of(origin) == host }
+            doomed.each { |origin| @db.delete("stores", origin) }
+            doomed.length
+          end
         end
 
         # @return [Integer, nil] seconds since the oldest entry was
         #   verified — nil when the index is empty. `doctor` and `index
         #   show` use this to say how stale the local index is.
         def oldest_verified_age(now: Time.now)
-          timestamps = entries.values.filter_map { |e| e["last_verified"] }
+          timestamps = all.filter_map { |e| e["last_verified"] }
           return nil if timestamps.empty?
 
           now.to_i - timestamps.min
         end
 
-        def exists? = File.exist?(@path)
+        def exists? = @db.exists?
 
         private
 
@@ -74,35 +79,6 @@ module Portage
         rescue URI::InvalidURIError
           ""
         end
-
-        def entries
-          @entries ||= read
-        end
-
-        def read
-          return {} unless File.readable?(@path)
-
-          # A business name/title can carry non-ASCII bytes (curly quotes,
-          # ®, accents); reading with the process's default external
-          # encoding (US-ASCII on a bare-minimal LANG, confirmed live in a
-          # sandbox with no locale set) would otherwise raise on the very
-          # first non-ASCII byte and get silently swallowed below, quietly
-          # dropping the whole file's worth of entries — this was caught by
-          # this phase's own live index-build check.
-          parsed = JSON.parse(File.read(@path, encoding: "UTF-8"))
-          parsed.is_a?(Hash) ? parsed : {}
-        rescue StandardError
-          {}
-        end
-
-        # An index that can't be written just means this run's findings
-        # aren't saved — never a failed build.
-        def write
-          FileUtils.mkdir_p(File.dirname(@path))
-          File.write(@path, JSON.generate(@entries))
-        rescue StandardError
-          nil
-        end
       end
     end
   end

data/lib/portage/cli/index.rb

@@ -1,5 +1,6 @@
 require_relative "index/store"
 require_relative "index/product_store"
+require_relative "index/entry_product"
 require_relative "index/known_cache"
 require_relative "index/sources"
 require_relative "index/builder"

data/lib/portage/cli/offer_sources.rb

@@ -49,6 +49,19 @@ module Portage
 
       def self.retail_handoff_host?(host) = HandoffOnly.matches_any?(host, RETAIL_HANDOFF_HOSTS)
 
+      # Adds the UCP Product wire hash to an offer as `product` (docs/plans/
+      # local-catalogue.md Phase 3), as the store served it, with `media`
+      # cut to the first image. Shared by Find and ShopifyCatalog so both
+      # trim the same way. Anything that isn't a non-empty hash leaves the
+      # offer as it was: the field is left out, not faked.
+      def self.with_product(offer, product)
+        return offer unless product.is_a?(Hash) && !product.empty?
+
+        media = product["media"]
+        product = product.merge("media" => media.first(1)) if media.is_a?(Array) && media.length > 1
+        offer.merge(product: product)
+      end
+
       # Shared by every retailer source below: an offer's `store` is the
       # origin of the item URL the API itself returned, same idea as
       # ShopifyCatalog#origin_of — kept as one module method instead of
@@ -133,8 +146,10 @@ module Portage
           return nil unless origin
 
           amount, currency = price_of(product)
-          { store: origin, source: name, checkout: nil, product_id: variant["id"], title: product["title"],
-            amount: amount, currency: currency, url: variant["url"] }
+          OfferSources.with_product(
+            { store: origin, source: name, checkout: nil, product_id: variant["id"], title: product["title"],
+              amount: amount, currency: currency, url: variant["url"] }, product
+          )
         end
 
         def origin_of(url)

data/lib/portage/cli/version.rb

@@ -1,5 +1,5 @@
 module Portage
   module Cli
-    VERSION = "0.10.0".freeze
+    VERSION = "0.12.0".freeze
   end
 end