portage-cli 0.9.0 → 0.11.0

data/lib/portage/cli/index/search.rb

@@ -0,0 +1,73 @@
+require "uri"
+
+module Portage
+  module Cli
+    module Index
+      # The SQL behind ProductStore#search: an FTS5 MATCH over products_fts
+      # (Index::Schema) ranked by bm25, or the same filters as a LIKE scan
+      # when the database has no FTS table.
+      module Search
+        # bm25 weights for products_fts's columns: key (unindexed), title,
+        # brand, category, aliases. A title hit outranks a brand-only one.
+        BM25 = "bm25(products_fts, 0.0, 10.0, 3.0, 1.0, 5.0)".freeze
+        LIKE_COLUMNS = %w[$.title $.brand $.category $.aliases].freeze
+        HOST_LIKE = Array.new(4) { "s.origin LIKE ?" }.join(" OR ").freeze
+
+        module_function
+
+        # Letters and digits only, so nothing the user types is FTS5 syntax.
+        # A trailing plural ending is dropped and each word matched as a
+        # prefix, so "lights" finds "Light" and "Lighting".
+        def words(query)
+          query.to_s.downcase.scan(/\p{Alnum}+/).map { |w| w.length > 3 ? w.sub(/(?:ies|es|s)\z/, "") : w }
+        end
+
+        def host_of(store)
+          return nil if store.to_s.strip.empty?
+
+          text = store.to_s.strip
+          (text.include?("://") ? URI.parse(text).host : text.split(%r{[/?]}).first).to_s.downcase
+        rescue URI::InvalidURIError
+          text.downcase
+        end
+
+        # @return [Array(String, Array)] sql and binds.
+        def sql(words, category:, host:, limit:, fts:)
+          match_sql, binds = fts ? fts_match(words) : like_match(words)
+          filters = [match_sql]
+          if category
+            filters << "json_extract(p.data, '$.category') = ?"
+            binds << category.to_s
+          end
+          if host
+            filters << "EXISTS (SELECT 1 FROM product_stores s WHERE s.key = p.key AND (#{HOST_LIKE}))"
+            binds.concat(host_patterns(host))
+          end
+          [select(fts, filters.join(" AND ")), binds + [limit.to_i]]
+        end
+
+        # The host itself or any subdomain of it, with or without a port —
+        # so `--store jbhifi.com.au` finds https://www.jbhifi.com.au.
+        def host_patterns(host) = ["%://#{host}", "%://#{host}:%", "%.#{host}", "%.#{host}:%"]
+
+        def select(fts, where)
+          if fts
+            "SELECT p.data FROM products_fts JOIN products p ON p.id = products_fts.rowid " \
+              "WHERE #{where} ORDER BY #{BM25} LIMIT ?"
+          else
+            "SELECT p.data FROM products p WHERE #{where} ORDER BY p.id LIMIT ?"
+          end
+        end
+
+        def fts_match(words)
+          ["products_fts MATCH ?", [words.map { |w| %("#{w}"*) }.join(" ")]]
+        end
+
+        def like_match(words)
+          any_column = LIKE_COLUMNS.map { |path| "lower(json_extract(p.data, '#{path}')) LIKE ?" }.join(" OR ")
+          [words.map { "(#{any_column})" }.join(" AND "), words.flat_map { |w| ["%#{w}%"] * LIKE_COLUMNS.length }]
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/index/sources/storefront_products/mapper.rb

@@ -0,0 +1,127 @@
+require "portage/ucp"
+
+require_relative "../../../classifier"
+
+module Portage
+  module Cli
+    module Index
+      module Sources
+        class StorefrontProducts
+          # One Shopify `/products.json` product -> Portage::Ucp::Product
+          # (docs/plans/local-catalogue.md Phase 2), then the index sighting
+          # taken from that Product. The product shape is UCP's, never a
+          # parallel "card" vocabulary.
+          #
+          # Not portage-ucp-shopify's Mapper: that one reads Storefront
+          # GraphQL nodes (camelCase, gids, MoneyV2 with a currency), not the
+          # REST products.json shape, and portage-cli doesn't depend on that
+          # gem.
+          #
+          # The Product carries what the store sent, price and availability
+          # included (products.json has no currency, so Price#currency is
+          # nil). #sighting is what the index persists, and it takes only
+          # identity fields from the Product: price and availability never
+          # reach it.
+          module Mapper
+            MAX_CATEGORIES = 3
+            TAXONOMY = "google_product_category".freeze
+            # Shopify's own placeholder for a product with no real options.
+            PLACEHOLDER_OPTION = { "name" => "Title", "values" => ["Default Title"] }.freeze
+
+            module_function
+
+            # @param raw [Hash] one entry of products.json's "products".
+            # @param classify [#call] text -> category ids (Classifier).
+            def product(raw, origin:, classify: Classifier.method(:categories_for))
+              options = product_options(raw)
+              Portage::Ucp::Product.new(
+                id: "gid://shopify/Product/#{raw['id']}", title: raw["title"].to_s,
+                description: Portage::Ucp::Description.new(html: raw["body_html"]),
+                price_range: price_range(raw), variants: Array(raw["variants"]).map { |v| variant(v, options) },
+                handle: raw["handle"], url: "#{origin}/products/#{raw['handle']}",
+                categories: categories(raw, classify), media: Array(raw["images"]).map { |i| media(i) },
+                options: options.map { |o| option(o) }, tags: Array(raw["tags"])
+              )
+            end
+
+            # @return [Hash] an Index::Builder sighting: the ProductStore
+            #   fields plus `product:`, the extra entry fields this source
+            #   adds (handle, url, image_url, options, variant_ids).
+            def sighting(raw, origin:, classify: Classifier.method(:categories_for))
+              product = product(raw, origin: origin, classify: classify)
+              { origin: origin, url: product.url, title: product.title, brand: brand(raw), gtin: nil,
+                categories: product.categories.select { |c| c.taxonomy == TAXONOMY }.map(&:value),
+                product: { handle: product.handle, url: product.url, image_url: product.media.first&.url,
+                           options: product.options.map(&:to_wire_h), variant_ids: product.variants.map(&:id) } }
+            end
+
+            def brand(raw)
+              vendor = raw["vendor"].to_s.strip
+              vendor.empty? ? nil : vendor
+            end
+
+            def product_options(raw)
+              Array(raw["options"]).reject { |o| o.slice("name", "values") == PLACEHOLDER_OPTION }
+            end
+
+            def option(raw)
+              values = Array(raw["values"]).map { |label| Portage::Ucp::OptionValue.new(label: label.to_s) }
+              Portage::Ucp::ProductOption.new(name: raw["name"].to_s, values: values)
+            end
+
+            def variant(raw, options)
+              Portage::Ucp::Variant.new(
+                id: "gid://shopify/ProductVariant/#{raw['id']}", title: raw["title"].to_s,
+                description: Portage::Ucp::Description.new(plain: raw["title"].to_s), price: price(raw["price"]),
+                sku: raw["sku"].to_s.empty? ? nil : raw["sku"],
+                list_price: raw["compare_at_price"] ? price(raw["compare_at_price"]) : nil,
+                availability: raw.key?("available") ? { "available" => raw["available"] } : nil,
+                options: selected_options(raw, options), media: variant_media(raw)
+              )
+            end
+
+            # option1..option3 line up with the product's options by
+            # position; the placeholder option is already gone, so a
+            # "Default Title" variant selects nothing.
+            def selected_options(raw, options)
+              options.each_with_index.filter_map do |opt, i|
+                label = raw["option#{i + 1}"]
+                Portage::Ucp::SelectedOption.new(name: opt["name"].to_s, label: label.to_s) if label
+              end
+            end
+
+            def variant_media(raw) = raw["featured_image"] ? [media(raw["featured_image"])] : []
+
+            def media(raw)
+              Portage::Ucp::Media.new(type: "image", url: raw["src"], alt_text: raw["alt"], width: raw["width"],
+                                      height: raw["height"])
+            end
+
+            def price(amount)
+              Portage::Ucp::Price.new(amount: Portage::Ucp::Support::Amounts.decimal_to_minor(amount), currency: nil)
+            end
+
+            def price_range(raw)
+              amounts = Array(raw["variants"]).map { |v| v["price"] }.compact
+              amounts = ["0"] if amounts.empty?
+              minors = amounts.map { |a| Portage::Ucp::Support::Amounts.decimal_to_minor(a) }
+              Portage::Ucp::PriceRange.new(min: Portage::Ucp::Price.new(amount: minors.min, currency: nil),
+                                           max: Portage::Ucp::Price.new(amount: minors.max, currency: nil))
+            end
+
+            # Classifier on product_type and tags (the plan's inputs), top
+            # three ids, then the store's own product_type as a merchant
+            # category.
+            def categories(raw, classify)
+              text = [raw["product_type"], *Array(raw["tags"])].compact.join(" ")
+              ids = text.strip.empty? ? [] : classify.call(text).first(MAX_CATEGORIES)
+              google = ids.map { |id| Portage::Ucp::Category.new(value: id, taxonomy: TAXONOMY) }
+              type = raw["product_type"].to_s.strip
+              type.empty? ? google : google + [Portage::Ucp::Category.new(value: type, taxonomy: "merchant")]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/index/sources/storefront_products/pages.rb

@@ -0,0 +1,114 @@
+require "net/http"
+require "uri"
+require "json"
+require "portage/ucp/support/connection"
+
+require_relative "../../../user_agent"
+
+module Portage
+  module Cli
+    module Index
+      module Sources
+        class StorefrontProducts
+          # The HTTP half of a crawl: GETs `products.json?limit=&page=`
+          # through Support::Connection with the shared UserAgent, pausing
+          # between pages, waiting out one 429's Retry-After and stopping on
+          # a second, and turning anything that isn't a products array into
+          # a reason.
+          class Pages
+            PAUSE = 1
+            MAX_RETRY_AFTER = 60
+            DEFAULT_RETRY_AFTER = 10
+            OPEN_TIMEOUT = 5
+            READ_TIMEOUT = 20
+
+            def self.get(url, accept: "application/json")
+              uri = URI.parse(url)
+              headers = UserAgent.headers.merge("Accept" => accept)
+              Portage::Ucp::Support::Connection.start(uri, route: :store, open_timeout: OPEN_TIMEOUT,
+                                                           read_timeout: READ_TIMEOUT) do |http|
+                http.get(uri.request_uri, headers)
+              end
+            end
+
+            # @param allowed [#call] path -> whether robots.txt allows it.
+            def initialize(endpoint, per_page:, max_pages:, sleeper:, allowed: ->(_path) { true })
+              @endpoint = endpoint
+              @path = URI.parse(endpoint).path
+              @allowed = allowed
+              @per_page = per_page
+              @max_pages = max_pages
+              @sleeper = sleeper
+              @rate_limited = 0
+            end
+
+            # Yields each non-empty page's raw products.
+            # @return [Array(String, String, Integer)] status ("ok",
+            #   "partial", "skipped"), reason (nil when ok) and pages read.
+            def each(&)
+              (1..@max_pages).each do |page|
+                @sleeper.call(PAUSE) if page > 1
+                products = fetch(page)
+                done = outcome(products, page, &)
+                return done if done
+              end
+              ["partial", "page_cap", @max_pages]
+            end
+
+            private
+
+            # nil means "full page, keep going".
+            def outcome(products, page)
+              return failed(products, page - 1) unless products.is_a?(Array)
+              return ["skipped", "empty", 0] if products.empty? && page == 1
+
+              yield products unless products.empty?
+              ["ok", nil, page] if products.length < @per_page
+            end
+
+            # Past page 1 a failure keeps what earlier pages found.
+            def failed(reason, pages) = [pages.zero? ? "skipped" : "partial", reason, pages]
+
+            # @return [Array<Hash>, String] a page's products, or why there
+            #   are none.
+            def fetch(page)
+              return "robots" unless @allowed.call("#{@path}#{query(page)}")
+
+              response = self.class.get("#{@endpoint}#{query(page)}")
+              return products_in(response) unless response.code == "429"
+
+              @rate_limited += 1
+              return "rate_limited" if @rate_limited > 1
+
+              @sleeper.call(retry_after(response["Retry-After"]))
+              fetch(page)
+            rescue StandardError => e
+              "error: #{e.class}"
+            end
+
+            # Page 1 is the bare URL (Shopify's default page), so a robots
+            # rule aimed at duplicate `?page=1` URLs doesn't read as a ban
+            # on the whole catalogue.
+            def query(page) = page == 1 ? "?limit=#{@per_page}" : "?limit=#{@per_page}&page=#{page}"
+
+            def products_in(response)
+              return "not_found" if response.code == "404"
+              return "redirect" if response.is_a?(Net::HTTPRedirection)
+              return "http_#{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+              body = JSON.parse(response.body.to_s.dup.force_encoding(Encoding::UTF_8))
+              body.is_a?(Hash) && body["products"].is_a?(Array) ? body["products"] : "not_json"
+            rescue JSON::ParserError
+              "not_json"
+            end
+
+            def retry_after(value)
+              seconds = Integer(value.to_s, exception: false) || DEFAULT_RETRY_AFTER
+              seconds.clamp(0, MAX_RETRY_AFTER)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/index/sources/storefront_products/robots.rb

@@ -0,0 +1,70 @@
+module Portage
+  module Cli
+    module Index
+      module Sources
+        class StorefrontProducts
+          # Just enough robots.txt (RFC 9309) to answer "may this agent GET
+          # this path?": the group naming this agent if there is one, else
+          # the `*` group; `*` and `$` in rules; the longest matching rule
+          # wins, and Allow wins a tie. No rules, or no file, means allowed.
+          class Robots
+            def initialize(body)
+              @groups = parse(body.to_s)
+            end
+
+            # @param path [String] request path, query included.
+            # @param agent [String] the User-Agent this process sends.
+            def allowed?(path, agent:)
+              rules = rules_for(agent.to_s.downcase)
+              matches = rules.select { |_allow, pattern| match?(pattern, path) }
+              return true if matches.empty?
+
+              longest = matches.map { |_allow, pattern| pattern.length }.max
+              matches.select { |_allow, pattern| pattern.length == longest }.any?(&:first)
+            end
+
+            private
+
+            def rules_for(agent)
+              named = @groups.select { |agents, *| agents.any? { |a| a != "*" && agent.include?(a) } }
+              chosen = named.empty? ? @groups.select { |agents, *| agents.include?("*") } : named
+              chosen.flat_map { |_agents, rules, _closed| rules }
+            end
+
+            # @return [Array<Array>] [agents, [[allow?, pattern], ...], closed]
+            #   per group.
+            def parse(body)
+              groups = []
+              body.each_line do |line|
+                field, value = line.sub(/#.*/, "").split(":", 2).map { |part| part.to_s.strip }
+                next if value.nil?
+
+                add_line(groups, field.downcase, value)
+              end
+              groups
+            end
+
+            # A rule line (even an empty `Disallow:`) ends a group's
+            # User-agent list, so the next User-agent starts a new group.
+            def add_line(groups, field, value)
+              if field == "user-agent"
+                groups << [[], [], false] if groups.empty? || groups.last[2]
+                groups.last[0] << value.downcase
+              elsif %w[allow disallow].include?(field) && groups.any?
+                groups.last[2] = true
+                groups.last[1] << [field == "allow", value] unless value.empty?
+              end
+            end
+
+            def match?(pattern, path)
+              anchored = pattern.end_with?("$")
+              body = anchored ? pattern.chomp("$") : pattern
+              regex = body.split("*", -1).map { |part| Regexp.escape(part) }.join(".*")
+              Regexp.new("\\A#{regex}#{'\\z' if anchored}").match?(path)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/index/sources/storefront_products.rb

@@ -0,0 +1,147 @@
+require "net/http"
+require "uri"
+
+require_relative "../../user_agent"
+require_relative "../../classifier"
+require_relative "../../handoff_only"
+require_relative "../store"
+require_relative "storefront_products/mapper"
+require_relative "storefront_products/robots"
+require_relative "storefront_products/pages"
+
+module Portage
+  module Cli
+    module Index
+      module Sources
+        # A store's own public catalogue, read from Shopify's `/products.json`
+        # (docs/plans/local-catalogue.md Phase 2), for origins already in
+        # the index. Off by default: it runs only from `portage index build
+        # --sources storefront_products` or `portage index add URL --crawl`,
+        # never from `find` or `buy`.
+        #
+        # Guardrails: a hand-off-only host (HandoffOnly, checked live) and a
+        # robots.txt Disallow get no products.json request at all; pages are
+        # MAX_PAGES x 250 at most, PAUSE seconds apart; a 429 waits out its
+        # Retry-After once and a second 429 stops that store; a 404, a
+        # redirect, an empty page 1, or anything that isn't products JSON
+        # (a bot wall) skips the store. Every outcome is noted on the store
+        # row as `crawl` ({status, reason, pages, products, at}).
+        #
+        # Yields Mapper sightings (no price, no availability) plus one store
+        # sighting per origin carrying `store_fields:` (the crawl note, and
+        # platform "shopify" when products.json answered); Index::Builder
+        # writes both.
+        class StorefrontProducts
+          MAX_PAGES = 20
+          MAX_STORES = 25
+          PER_PAGE = 250
+
+          # The one platform seam: WooCommerce's Store API can join here.
+          # nil platform means "not known yet", worth a try.
+          def self.endpoint_for(origin, platform)
+            "#{origin}/products.json" if platform.nil? || platform == "shopify"
+          end
+
+          # The guardrails, plus test seams for sleeping and the clock.
+          def initialize(stores: Store.new, handoff_only: HandoffOnly.new, max_pages: MAX_PAGES,
+                         max_stores: MAX_STORES, per_page: PER_PAGE, sleeper: Kernel.method(:sleep), now: Time.now)
+            @stores = stores
+            @handoff_only = handoff_only
+            @max_pages = max_pages
+            @max_stores = max_stores
+            @per_page = per_page
+            @sleeper = sleeper
+            @now = now
+          end
+
+          def name = "storefront_products"
+
+          def description
+            "Each indexed store's own /products.json (Shopify), up to #{MAX_PAGES} pages a store and " \
+              "#{MAX_STORES} stores a run, least recently crawled first. Off by default — opt in with " \
+              "--sources storefront_products or `index add URL --crawl`."
+          end
+
+          def source_path = nil
+
+          # `**` accepts (and ignores) the shared Source#candidates(queries:)
+          # interface.
+          def candidates(**)
+            eligible = @stores.all.select do |entry|
+              self.class.endpoint_for(entry["origin"], entry["platform"]) && !handoff_only?(entry["origin"])
+            end
+            eligible.sort_by { |entry| entry.dig("crawl", "at").to_i }.first(@max_stores)
+                    .flat_map { |entry| crawl(entry["origin"], platform: entry["platform"]) }
+          end
+
+          # @return [Array<Hash>] product sightings, then the store sighting.
+          def crawl(origin, platform: nil)
+            endpoint = self.class.endpoint_for(origin, platform)
+            robots = robots_for(origin, endpoint)
+            return [store_sighting(origin, note("skipped", robots, 0, 0))] if robots.is_a?(String)
+
+            sightings = []
+            classify = memoized_classifier
+            status, reason, pages = pages_for(endpoint, robots).each do |products|
+              sightings.concat(products.map { |raw| Mapper.sighting(raw, origin: origin, classify: classify) })
+            end
+            sightings + [store_sighting(origin, note(status, reason, pages, sightings.length))]
+          rescue StandardError => e
+            [store_sighting(origin, note("skipped", "error: #{e.class}", 0, 0))]
+          end
+
+          private
+
+          def pages_for(endpoint, robots)
+            Pages.new(endpoint, per_page: @per_page, max_pages: @max_pages, sleeper: @sleeper,
+                                allowed: ->(path) { robots.allowed?(path, agent: UserAgent.value) })
+          end
+
+          # The store's robots.txt rules, or why the crawl stops before
+          # any request (no endpoint, hand-off only) or before any
+          # products.json request (robots.txt unreachable). Pages checks
+          # the rules against every page URL it would request.
+          # RFC 9309: a 4xx robots.txt means no rules; a 5xx or no answer
+          # at all means stay out.
+          # @return [Robots, String]
+          def robots_for(origin, endpoint)
+            return "unsupported_platform" unless endpoint
+            return "handoff_only" if handoff_only?(origin)
+
+            response = Pages.get("#{origin}/robots.txt", accept: "text/plain")
+            return Robots.new(nil) if response.code.start_with?("4")
+            return "redirect" if response.is_a?(Net::HTTPRedirection)
+            return "robots_unreachable" unless response.is_a?(Net::HTTPSuccess)
+
+            Robots.new(response.body.to_s.dup.force_encoding(Encoding::UTF_8))
+          rescue StandardError
+            "robots_unreachable"
+          end
+
+          # Product types and tags repeat across a catalogue, and Classifier
+          # re-reads its YAML on every call.
+          def memoized_classifier
+            memo = {}
+            ->(text) { memo[text] ||= Classifier.categories_for(text) }
+          end
+
+          def handoff_only?(origin)
+            @handoff_only.host?(URI.parse(origin.to_s).host)
+          rescue URI::InvalidURIError
+            true
+          end
+
+          def note(status, reason, pages, products)
+            { "status" => status, "reason" => reason, "pages" => pages, "products" => products, "at" => @now.to_i }
+          end
+
+          def store_sighting(origin, crawl)
+            fields = { crawl: crawl }
+            fields[:platform] = "shopify" unless crawl["status"] == "skipped"
+            { origin: origin, url: nil, title: nil, brand: nil, gtin: nil, store_fields: fields }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/portage/cli/index/sources.rb

@@ -3,6 +3,7 @@ require_relative "sources/stores_file"
 require_relative "sources/browser"
 require_relative "sources/wikidata"
 require_relative "sources/webmcp_sweep"
+require_relative "sources/storefront_products"
 
 module Portage
   module Cli
@@ -16,7 +17,8 @@ module Portage
           "stores_file" => -> { StoresFile.new },
           "browser" => -> { Browser.new },
           "wikidata" => -> { Wikidata.new },
-          "webmcp_sweep" => -> { WebmcpSweep.new }
+          "webmcp_sweep" => -> { WebmcpSweep.new },
+          "storefront_products" => -> { StorefrontProducts.new }
         }.freeze
 
         # Runs with no extra opt-in: no bridge required, and no low-yield
@@ -24,7 +26,8 @@ module Portage
         # show up in `portage index sources`, just not run unless named in
         # `--sources`. `browser` is listed but never a default — it yields
         # nothing; `portage browser import` writes those entries itself
-        # (see Sources::Browser).
+        # (see Sources::Browser). `storefront_products` sends up to 21
+        # requests a store, so it runs only when named.
         DEFAULT_NAMES = %w[shopify_catalog stores_file].freeze
 
         def self.all = ALL.values.map(&:call)