html2rss 0.30.0 → 0.32.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d839914f2d4d26e86998293a431a76168033523857ec8bc65f3f447441cb07e
-  data.tar.gz: fd0008a8d7a527b0a2128df3a5603a06c154d61b522f6a14decded9889615b78
+  metadata.gz: 4841672d87f3828928fa35646176d878469ade0afce9f9e6a119ccd1cf3251ae
+  data.tar.gz: 618382810f59ff95713064d91a2ccb6df4c0dd2cf6287f31127cddd5848e8dbd
 SHA512:
-  metadata.gz: 418b9fd5973f19985d7e80abb7734530689bff0753ec948813907f35fd1d2a139ec9fd26353c8c4c87ffc456f446be4a9cdfee9c406022fdfb6c6f0077f886fa
-  data.tar.gz: 1a470470f9bdaea585fdccd56e231c3308624e745a98b830bfd1f0861dad4ce22a98c4a2216306cfd893d6d271efc1d68c6efe15ea6e015d43776086db3094ad
+  metadata.gz: 7abf8800a5943fca86ed9fa19058a1957ed101d52db70fd75042bda91abc2a9070d3bb5e6b0129e18103ebf407de2ea97a5e1a826cec3b5a7c0f21c12d0dfd61
+  data.tar.gz: 7831958f9d9b2af2db632b341cd8c902b45ed78feaf12dc21ee4f9c80c72237d56acba58e1b2aa7b0c842fc6dd3626beb48fd0e08ef932730920e33c23eb2f4b

data/README.md

@@ -1,4 +1,4 @@
-![html2rss logo](https://github.com/html2rss/html2rss/raw/master/support/logo.png)
+![html2rss logo](https://html2rss.github.io/assets/logo.png)
 
 [![Gem Version](https://badge.fury.io/rb/html2rss.svg)](http://rubygems.org/gems/html2rss) [![Yard Docs](http://img.shields.io/badge/yard-docs-blue.svg)](https://www.rubydoc.info/gems/html2rss) ![Retro Badge: valid RSS](https://validator.w3.org/feed/images/valid-rss-rogers.png) [![CI](https://github.com/html2rss/html2rss/workflows/lint%20and%20test/badge.svg)](https://github.com/html2rss/html2rss/actions)
 
@@ -62,6 +62,9 @@ html2rss capture https://example.com/news | html2rss test -
 # One-shot articles now
 html2rss scrape https://example.com/news
 html2rss scrape https://example.com/news --format jsonfeed --explain
+# Off-domain item URLs are dropped by default. --keep-different-domain opts in
+# and lets the remote source control published links to unrelated domains.
+html2rss scrape https://example.com/app-list.json --keep-different-domain
 
 # Durable config workflow
 html2rss capture https://example.com/news --write feed.yml
@@ -82,7 +85,7 @@ Historic CLI aliases: `feed` → `apply`, `auto` → `scrape`.
 
 Inspect follows redirects and reports the landing URL in `final_url`. CLI text shows a `Final:` line **only when** the landing URL differs from what you typed — that line means the redirect succeeded, not that inspect stopped early.
 
-Cross-host redirects (e.g. `https://apex.example/` → `https://www.example/`) set `Host` per hop via Faraday/Net::HTTP; html2rss does not pin the entry hostname. When `final_url` differs and status is 4xx, retry on `final_url` or pass the site's canonical hostname. Details: [`lib/html2rss/page_recon/README.md`](lib/html2rss/page_recon/README.md).
+Cross-host redirects (e.g. `https://apex.example/` → `https://www.example/`) set `Host` per hop via HTTPX; html2rss does not pin the entry hostname. When `final_url` differs and status is 4xx, retry on `final_url` or pass the site's canonical hostname. Details: [`lib/html2rss/page_recon/README.md`](lib/html2rss/page_recon/README.md).
 
 ## Capture API
 
@@ -112,7 +115,7 @@ stdio uses stdout for JSON-RPC, so the daemon logs to **stderr**. It defaults to
 
 HTTP transport needs `rack`, `rackup`, and `webrick` (declared gem dependencies). It listens on `127.0.0.1` only; do not expose it on a public interface without your own auth and Host/Origin controls.
 
-**Strategy note:** MCP `scrape` / `capture` with `strategy: "auto"` run Faraday → Botasaurus AutoFallback. `inspect` uses Faraday when `auto` (cheap diagnostic); pin `botasaurus` when you need browser rendering for inspect.
+**Strategy note:** MCP `scrape` / `capture` with `strategy: "auto"` run default (HTTPX) → Botasaurus AutoFallback. `inspect` uses default (HTTPX) when `auto` (cheap diagnostic); pin `botasaurus` when you need browser rendering for inspect.
 
 **Tool-call budget:** `scrape` is 1 call (auto already hops). Durable config is `capture` → `test` → `apply` (or `validate` → `test` → `apply` when you already have YAML). Call `inspect` only when scrape/capture is weak or you need recon (final URL, status, https→http, native RSS/Atom).
 

data/lib/html2rss/article/deduplicator.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'set' # rubocop:disable Lint/RedundantRequireStatement
-
 module Html2rss
   class Article
     ##
@@ -40,9 +38,9 @@ module Html2rss
       attr_reader :articles
 
       def deduplication_fingerprint_for(article)
-        return unless article.respond_to?(:deduplication_fingerprint)
-
         article.deduplication_fingerprint
+      rescue NoMethodError
+        nil
       end
     end
   end

data/lib/html2rss/article.rb

@@ -5,14 +5,9 @@ require 'zlib'
 module Html2rss
   ##
   # Article is a simple data object representing an article extracted from a page.
-  # It is enumerable and responds to all keys specified in PROVIDED_KEYS.
   #
   # Description and enclosure wire presentation live in {FeedBuilder::ItemPresentation}.
-  # rubocop:disable-next Metrics/ClassLength -- value object retains Marshal + defensive freeze helpers
   class Article
-    include Enumerable
-    include Comparable
-
     # Allowed article attributes accepted by the value object constructor.
     PROVIDED_KEYS = %i[id title description url image author guid published_at enclosures categories scraper].freeze
     # Separator used to build deterministic deduplication fingerprints.
@@ -23,26 +18,24 @@ module Html2rss
     # when attributes are lazily/conditionally accessed in different sequences.
     NOT_SET = Object.new.freeze
 
-    # @param options [Hash{Symbol => String}]
-    # @option options [String] :id stable article identifier
-    # @option options [String] :title article title
-    # @option options [String] :description raw extracted description/content (not feed-rendered HTML)
-    # @option options [String, Html2rss::Url] :url canonical article URL
-    # @option options [String, Html2rss::Url] :image image URL for description / JSON Feed +image+
-    # @option options [String] :author author name
-    # @option options [String] :guid explicit GUID override
-    # @option options [String, Time, DateTime] :published_at publication timestamp
-    # @option options [Array<Hash{Symbol => Object}>] :enclosures enclosure attribute hashes
-    # @option options [Array<String>] :categories category labels
-    # @option options [Class] :scraper scraper class that produced the article
-    def initialize(**options)
-      @to_h = options.each_with_object({}) { |(key, value), hash| hash[key] = freeze_option(value) }.freeze
+    # @param id [String, nil] stable article identifier
+    # @param title [String, nil] article title
+    # @param description [String, nil] raw extracted description/content (not feed-rendered HTML)
+    # @param url [String, Html2rss::Url, nil] canonical article URL
+    # @param image [String, Html2rss::Url, nil] image URL for description / JSON Feed +image+
+    # @param author [String, nil] author name
+    # @param guid [String, Array, nil] explicit GUID override
+    # @param published_at [String, Time, DateTime, nil] publication timestamp
+    # @param enclosures [Array<Hash{Symbol => Object}>, nil] enclosure attribute hashes
+    # @param categories [Array<String>, nil] category labels
+    # @param scraper [Class, nil] scraper class that produced the article
+    # rubocop:disable-next Metrics/ParameterLists -- eleven article keys stay explicit
+    def initialize(id: nil, title: nil, description: nil, url: nil, image: nil, author: nil, guid: nil,
+                   published_at: nil, enclosures: nil, categories: nil, scraper: nil)
+      @to_h = { id:, title:, description:, url:, image:, author:, guid:, published_at:, enclosures:, categories:,
+                scraper: }.compact.transform_values { freeze_option(_1) }.freeze
 
       @url = @image = @guid = @enclosures = @categories = @published_at = NOT_SET
-
-      return unless (unknown_keys = options.keys - PROVIDED_KEYS).any?
-
-      Log.warn "Article: unknown keys found: #{unknown_keys.join(', ')}"
     end
 
     # Checks if the article is valid based on the presence of URL, ID, and either title or description.
@@ -51,14 +44,6 @@ module Html2rss
       !url.to_s.empty? && (!title.to_s.empty? || !description.to_s.empty?) && !id.to_s.empty?
     end
 
-    # @yield [key, value]
-    # @return [Enumerator] if no block is given
-    def each
-      return enum_for(:each) unless block_given?
-
-      PROVIDED_KEYS.each { |key| yield(key, public_send(key)) }
-    end
-
     # @return [String, nil] stable article identifier
     def id = blank_string_to_nil(@to_h[:id])
 
@@ -139,14 +124,6 @@ module Html2rss
       @to_h[:scraper]
     end
 
-    # @param other [Object] value compared against this article
-    # @return [Integer, nil] comparison result for compatible Article values
-    def <=>(other)
-      return nil unless other.is_a?(Article)
-
-      0 if other.all? { |key, value| value == public_send(key) ? public_send(key) <=> value : false }
-    end
-
     private
 
     ##

data/lib/html2rss/auto_source/README.md

@@ -10,8 +10,9 @@ Entry: `FeedPipeline` → `AutoSource#articles` → `Scraper.build_instance` →
 
 ## Live flow
 
-1. **Request** — `RequestSession` returns a `Response` with `body` (String) and `parsed_body` (Nokogiri HTML). Direct syndication responses skip HTML scrapers and parse via `Syndication::Parser`.
-2. **Scraper tiers** — Enabled scrapers that claim the page (shallow `articles?` or instance `extractable?`) run in `Scraper::SCRAPER_TIERS` order. Merge within a tier, then stop when enough articles survive Cleanup:
+1. **Request** — `RequestSession` returns a `Response` with `body` (String) and `parsed_body` (Nokogiri HTML, or parsed JSON for JSON media types). Direct syndication responses skip HTML scrapers and parse via `Syndication::Parser`.
+2. **JSON document body** — When the response is a JSON media type (`Response#json_response?`), AutoSource runs `JsonDocument` only (no HTML tiers). Scalar JSON is `unsupported_surface`. Same-domain Cleanup stays on by default (`keep_different_domain: false`). If the scraper yielded items and Cleanup kept none, `Log.warn` emits `auto_source.admission_emptied` with drop tallies; a `different_domain` tally hints at `auto_source.cleanup.keep_different_domain: true` / `--keep-different-domain`, which lets the remote source control off-domain link targets (no affiliation or safety check). Guidance stays in help/docs and that structured warning — stdout feed output stays composable.
+3. **Scraper tiers** (HTML only) — Enabled scrapers that claim the page (shallow `articles?` or instance `extractable?`) run in `Scraper::SCRAPER_TIERS` order. Merge within a tier, then stop when enough articles survive Cleanup:
 
    0. Native feed: NativeFeed (head alternates + path discovery → RSS/Atom parse)
    1. In-page structured: Schema, Microdata, Microformats2, JsonState, XhrArticles
@@ -19,22 +20,23 @@ Entry: `FeedPipeline` → `AutoSource#articles` → `Scraper.build_instance` →
    3. Heuristic: SemanticHtml
    4. Heuristic: Html (skipped when earlier tiers already admitted at least one clean article)
 
-3. **Structured / API scrapers** — Schema, Microdata, MF2, JSON state, XHR JSON, oEmbed, WordPress REST, and Sitemap work on Nokogiri CSS/XPath or JSON/XML parsers. They do not use SST.
-4. **Heuristic scrapers** — `SemanticHtml` and `Html` normalize once into an `SST::Document`, then:
+4. **Structured / API scrapers** — Schema, Microdata, MF2, JSON state, XHR JSON, oEmbed, WordPress REST, and Sitemap work on Nokogiri CSS/XPath or JSON/XML parsers. They do not use SST. `JsonDocument` walks a JSON response body (JSON Feed mapping or `JsonState.discover_articles`).
+5. **Heuristic scrapers** — `SemanticHtml` and `Html` normalize once into an `SST::Document`, then:
 
    `SST::Normalizer` → `AutoSource::Segmenter` → `Scoring::Engine` → extractor / article materialization.
 
-5. **Cleanup** — Merge, dedupe, hard-exclude non-article destinations (via `PathClassifier` facts), drop junk titles, and trim to `limit`. Html is skipped when earlier tiers already admitted clean items.
+6. **Cleanup** — Merge, dedupe, hard-exclude non-article destinations (via `PathClassifier` facts), drop junk titles, and trim to `limit`. Html is skipped when earlier tiers already admitted clean items.
 
-6. **Entry URL resolution** (pipeline, not this class) — when AutoFallback sees a weak homepage extract, `FeedResolution` may rewrite the scrape URL to a listing/feed before escalating strategies. See {Html2rss::FeedPipeline} and {Html2rss::FeedResolution}.
+7. **Entry URL resolution** (pipeline, not this class) — when AutoFallback sees a weak homepage extract, `FeedResolution` may rewrite the scrape URL to a listing/feed before escalating strategies. See {Html2rss::FeedPipeline} and {Html2rss::FeedResolution}.
 
-Segmenter strategies: `:semantic` (leaf containers + primary link), `:list` (repeated tag paths), `:cluster` (class/structure grids for anchorless cards). Scoring ranks and demotes; `LinkDestination::NoisePolicy` owns content-anchor eligibility. Cleanup owns feed-item admission.
+Segmenter strategies: `:semantic` (leaf containers + primary link), `:list` (cards grouped by parent; `main` is a filter), `:cluster` (class/structure grids for anchorless cards). Scoring ranks and demotes; `LinkDestination::NoisePolicy` owns content-anchor eligibility. Cleanup owns feed-item admission.
 
 ## Nokogiri vs SST boundaries
 
 | Surface                                                                                       | Owns DOM                                                         |
 | --------------------------------------------------------------------------------------------- | ---------------------------------------------------------------- |
-| `Response#parsed_body`                                                                        | Single HTML parse for the page                                   |
+| `Response#parsed_body`                                                                        | HTML document, or Hash/Array for JSON bodies                     |
+| `JsonDocument`                                                                                | Parsed JSON Hash/Array (no Nokogiri)                             |
 | Schema / Microdata / MF2 / JsonState / XhrArticles / MetaOembed / app-shell detection         | Nokogiri                                                         |
 | Sitemap detection (CSS/XPath)                                                                 | Nokogiri; URL list parsing uses raw `response.body` (XML string) |
 | Selectors path / Sanitize transformers                                                        | Nokogiri (unchanged)                                             |
@@ -51,7 +53,8 @@ Production heuristic scrapers should reuse one `SST::Document` memoized from `pa
 
 ## Non-goals
 
-- Replacing Selectors, Schema, Microdata, MF2, JsonState, or XhrArticles with SST.
+- Replacing Selectors, Schema, Microdata, MF2, JsonState, JsonDocument, or XhrArticles with SST.
+- Turning `keep_different_domain` on automatically for JSON documents (operators must opt in explicitly; the flag trusts the source for any published link domain).
 - A second HTML parser beside Nokogiri.
 - Dual Response payload (Nokogiri + SST always).
 - App-shell classification on SST.

data/lib/html2rss/auto_source/cleanup.rb

@@ -17,7 +17,7 @@ module Html2rss
       MIN_WORDS = 3
 
       # Allowed URL schemes for article filtering.
-      VALID_SCHEMES = %w[http https].to_set.freeze
+      VALID_SCHEMES = Set['http', 'https'].freeze
 
       # Photo-credit agencies (single list → junk title regexes).
       CREDIT_AGENCIES = [
@@ -63,10 +63,28 @@ module Html2rss
       # Read-only ship-quality audit on RSS/article-like items (no mutation).
       AuditResult = Data.define(:warnings, :metrics, :violations)
 
+      # Config key / CLI flag named when different_domain drops emptied admission.
+      # Opt-in trusts the remote source for off-domain link targets (no affiliation check).
+      DIFFERENT_DOMAIN_HINT =
+        'auto_source.cleanup.keep_different_domain: true (or --keep-different-domain); ' \
+        'allows published items to link to any domain supplied by the source'
+
       class << self
+        # Mitigating config hint when admission drops emptied a non-empty capture.
+        # Other drop reasons remain counts only.
+        #
+        # @param drop_tallies [Hash{String => Integer}]
+        # @return [String, nil]
+        def admission_hint(drop_tallies)
+          return unless drop_tallies.fetch('different_domain', 0).positive?
+
+          DIFFERENT_DOMAIN_HINT
+        end
+
         # @param articles [Array<Article>] extracted article candidates
         # @param url [Html2rss::Url] feed source URL used for same-host filtering
-        # @param keep_different_domain [Boolean] whether to keep off-domain entries
+        # @param keep_different_domain [Boolean] when true, keep items whose URLs are on a
+        #   different domain than +url+ (source-controlled off-domain links; default false)
         # @return [Result] cleaned articles and frozen drop tallies
         def call(articles, url:, keep_different_domain: DEFAULT_CONFIG.fetch(:keep_different_domain)) # rubocop:disable Metrics/MethodLength -- ordered reject pipeline
           Log.debug "Cleanup: start with #{articles.size} articles"
@@ -97,13 +115,14 @@ module Html2rss
           JUNK_TITLE_RULES.find { |_, pattern| pattern.match?(normalized) }&.first
         end
 
-        # @param items [Array] RSS or article-like objects with title and link/url
+        # @param items [Array<Html2rss::Article, RSS::Rss::Channel::Item>]
         # @return [AuditResult]
         def audit_feed_items(items)
-          metrics = initial_audit_metrics(items.size)
+          articles = items.map { |item| feed_item_as_article(item) }
+          metrics = initial_audit_metrics(articles.size)
           violations = Hash.new(0)
-          warnings = audit_url_diversity(items, metrics, violations)
-          audit_item_titles(items, metrics, violations)
+          warnings = audit_url_diversity(articles, metrics, violations)
+          audit_item_titles(articles, metrics, violations)
           warnings = finalize_audit_warnings(warnings, metrics)
           log_audit(metrics, warnings, violations)
           AuditResult.new(warnings:, metrics: metrics.freeze, violations: violations.freeze)
@@ -256,8 +275,16 @@ module Html2rss
           url&.without_fragment&.to_s
         end
 
+        def feed_item_as_article(item)
+          case item
+          when Article then item
+          else
+            Article.new(title: item.title, url: item.link)
+          end
+        end
+
         def item_url(item)
-          raw = item.respond_to?(:link) ? item.link : item.url
+          raw = item.url
           raw.nil? || raw.to_s.empty? ? nil : Html2rss::Url.from_absolute(raw.to_s)
         rescue ArgumentError
           nil

data/lib/html2rss/auto_source/scraper/html.rb

@@ -22,6 +22,11 @@ module Html2rss
         # @return [Symbol]
         def self.options_key = :html
 
+        ##
+        # @param _opts [Hash] unused options
+        # @return [Integer]
+        def self.request_slots(_opts = {}) = 0
+
         ##
         # @param parsed_body [Nokogiri::HTML::Document]
         # @return [Boolean]

data/lib/html2rss/auto_source/scraper/json_document.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      # Extracts articles from a parsed JSON response body (Hash or Array).
+      # JSON Feed documents map item fields directly; every other document walks
+      # via {JsonState.discover_articles}. Does not scan HTML.
+      class JsonDocument
+        include Enumerable
+
+        # Exact JSON Feed 1.0 / 1.1 +version+ URLs (no substring host matching).
+        JSON_FEED_VERSIONS = Set[
+          'https://jsonfeed.org/version/1',
+          'https://jsonfeed.org/version/1.1'
+        ].freeze
+
+        # @return [Symbol] scraper config key
+        def self.options_key = :json_document
+
+        # @param _opts [Hash] unused scraper options
+        # @return [Integer] zero — no additional HTTP requests
+        def self.request_slots(_opts = {}) = 0
+
+        # Not detectable from HTML alone; instance {#extractable?} requires a
+        # Hash or Array body (wired by AutoSource for JSON responses).
+        #
+        # @param _parsed_body [Object] unused
+        # @return [Boolean]
+        def self.articles?(_parsed_body) = false
+
+        # @param parsed_body [Hash, Array, Object] parsed JSON document (or HTML when misrouted)
+        # @param url [String, Html2rss::Url] base URL for relative link resolution
+        # @param _opts [Hash] scraper-specific options
+        # @option _opts [Object] :_reserved reserved for future scraper-specific options
+        def initialize(parsed_body, url:, **_opts)
+          @parsed_body = parsed_body
+          @url = url
+        end
+
+        # @return [Boolean] true when +parsed_body+ is a Hash or Array
+        def extractable?
+          @parsed_body.is_a?(Hash) || @parsed_body.is_a?(Array)
+        end
+
+        # @yield [Hash{Symbol => Object}] normalized article hash
+        # @return [Enumerator, void] article enumerator when no block is given
+        def each(&block)
+          return enum_for(:each) unless block
+          return unless extractable?
+
+          if json_feed?
+            each_json_feed_item(&block)
+          else
+            JsonState.discover_articles(@parsed_body, base_url: @url) do |article|
+              yield(article) if article
+            end
+          end
+        end
+
+        private
+
+        def json_feed?
+          return false unless @parsed_body.is_a?(Hash)
+
+          JSON_FEED_VERSIONS.include?(@parsed_body[:version].to_s) && @parsed_body[:items].is_a?(Array)
+        end
+
+        def each_json_feed_item
+          @parsed_body[:items].each do |item|
+            article = map_json_feed_item(item)
+            yield article if article
+          end
+        end
+
+        def map_json_feed_item(item)
+          return unless item.is_a?(Hash)
+
+          article_url = resolve_json_feed_url(item)
+          return unless article_url
+
+          json_feed_article_hash(item, article_url)
+        end
+
+        def json_feed_article_hash(item, article_url)
+          {
+            title: string(item[:title]),
+            description: json_feed_description(item),
+            url: article_url,
+            image: resolve_optional_url(item[:image]),
+            published_at: string(item[:date_published]),
+            categories: json_feed_tags(item[:tags]),
+            id: string(item[:id]) || article_url.to_s
+          }.compact
+        end
+
+        def resolve_json_feed_url(item)
+          resolve_optional_url(item[:url]) || absolute_url(item[:id])
+        end
+
+        def resolve_optional_url(value)
+          str = string(value)
+          return unless str
+
+          Url.from_relative(str, @url)
+        rescue ArgumentError
+          nil
+        end
+
+        def absolute_url(value)
+          str = string(value)
+          return unless str
+
+          Url.from_absolute(str)
+        rescue ArgumentError
+          nil
+        end
+
+        def json_feed_description(item)
+          string(item[:content_text]) || string(item[:content_html])
+        end
+
+        def json_feed_tags(tags)
+          return unless tags.is_a?(Array)
+
+          names = tags.filter_map { |tag| string(tag) }
+          names.uniq!
+          names unless names.empty?
+        end
+
+        def string(value) = JsonState::ArticleNormalizer.string(value)
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb

@@ -5,15 +5,15 @@ module Html2rss
     module Scraper
       class JsonState
         # Shapes raw entries into the structure required downstream.
-        module ArticleNormalizer
+        module ArticleNormalizer # rubocop:disable Metrics/ModuleLength -- field keys + URL/scalar unwrap stay co-located
           # Preferred keys when extracting title-like values from state payloads.
           TITLE_KEYS = %i[title headline name text].freeze
           # Preferred keys when extracting URL-like values from state payloads.
-          URL_KEYS = %i[url link href permalink slug path canonicalUrl shortUrl].freeze
+          URL_KEYS = %i[url link href permalink slug path canonicalUrl shortUrl alias self].freeze
           # Preferred keys when extracting description-like values from state payloads.
-          DESCRIPTION_KEYS = %i[description summary excerpt dek subheading].freeze
+          DESCRIPTION_KEYS = %i[description summary excerpt dek subheading abstract].freeze
           # Preferred keys when extracting image-like values from state payloads.
-          IMAGE_KEYS = %i[image imageUrl thumbnailUrl thumbnail src featuredImage coverImage heroImage].freeze
+          IMAGE_KEYS = %i[image imageUrl thumbnailUrl thumbnail src featuredImage coverImage heroImage icon].freeze
           # Preferred keys when extracting publication timestamps from state payloads.
           PUBLISHED_AT_KEYS = %i[published_at publishedAt datePublished date publicationDate pubDate updatedAt
                                  updated_at createdAt created_at].freeze
@@ -21,6 +21,12 @@ module Html2rss
           CATEGORY_KEYS = %i[categories tags section sections topic topics channel].freeze
           # Preferred keys when extracting identifier-like values from state payloads.
           ID_KEYS = %i[id guid uuid slug key].freeze
+          # Max times to unwrap a Hash URL value (e.g. path.alias, links.self.href).
+          MAX_URL_UNWRAP = 3
+          # Nested title/description keys (WordPress +rendered+, CMS +value+/+text+).
+          SCALAR_KEYS = %i[rendered value text].freeze
+          # Snake `_at`/`_on` or camel `At`/`On` suffix after a closed published-at miss.
+          DATE_KEY_SUFFIX = /_at\z|_on\z|[a-z]At\z|[a-z]On\z/
 
           module_function
 
@@ -44,37 +50,94 @@ module Html2rss
               url: article_url,
               image: resolve_link(entry, keys: IMAGE_KEYS, base_url:,
                                          log_key: 'JsonState: invalid image URL encountered'),
-              published_at: string(ValueFinder.fetch(entry, PUBLISHED_AT_KEYS)),
+              published_at: published_at(entry),
               categories: categories(entry),
               id: identifier(entry, article_url)
             }.compact
           end
           # rubocop:enable Metrics/MethodLength
 
-          # @param value [Object] candidate scalar value
-          # @return [String, nil] normalized non-empty string value
+          # @param value [Object] candidate scalar, or Hash wrapping +rendered+/+value+/+text+
+          # @return [String, nil] normalized non-empty string (HTML preserved for descriptions)
           def string(value)
+            value = unwrap_scalar(value)
+            return unless value.is_a?(String) || value.is_a?(Numeric)
+
             trimmed = value.to_s.strip
             trimmed unless trimmed.empty?
           end
 
+          # @param value [Object] candidate scalar or nested Hash
+          # @return [Object, nil] String/Numeric after at most {MAX_URL_UNWRAP} Hash unwraps
+          def unwrap_scalar(value)
+            depth = 0
+            while value.is_a?(Hash) && depth < MAX_URL_UNWRAP
+              nested = SCALAR_KEYS.find { |key| value.key?(key) }
+              break unless nested
+
+              value = value[nested]
+              depth += 1
+            end
+            value unless value.is_a?(Hash) || value.is_a?(Array)
+          end
+
           # @param entry [Hash] raw article entry candidate
-          # @param keys [Array<String>] preferred link keys
+          # @param keys [Array<Symbol>] preferred link keys
           # @param base_url [String, Html2rss::Url] base URL for relative link resolution
           # @param log_key [String] structured log message key
           # @return [Html2rss::Url, nil] resolved absolute URL
           def resolve_link(entry, keys:, base_url:, log_key:)
-            value = ValueFinder.fetch(entry, keys)
-            value = ValueFinder.fetch(value, keys) if value.is_a?(Hash)
+            value = unwrap_url(ValueFinder.fetch(entry, keys), keys) ||
+                    unwrap_url(ValueFinder.fetch(entry[:links], keys), keys)
             string = string(value)
             return unless string
 
             Url.from_relative(string, base_url)
           rescue ArgumentError
-            Log.debug(log_key, url: string)
+            Log.debug("#{log_key}: url=#{string}")
             nil
           end
 
+          # @param value [Object] candidate URL value, possibly a nested Hash
+          # @param keys [Array<Symbol>] preferred link keys
+          # @return [Object, nil] scalar URL candidate after at most {MAX_URL_UNWRAP} Hash unwraps
+          def unwrap_url(value, keys)
+            depth = 0
+            while value.is_a?(Hash) && depth < MAX_URL_UNWRAP
+              value = ValueFinder.fetch(value, keys)
+              depth += 1
+            end
+            value unless value.is_a?(Hash)
+          end
+
+          # @param entry [Hash] raw article entry candidate
+          # @return [String, nil] publication timestamp string or unix timestamp digits
+          def published_at(entry)
+            value = ValueFinder.fetch(entry, PUBLISHED_AT_KEYS)
+            string(value) || string(suffix_date_value(entry))
+          end
+
+          # @param hash [Hash, nil] entry or nested attributes/data hash
+          # @return [Object, nil] first string or numeric value whose key matches {DATE_KEY_SUFFIX}
+          def suffix_date_value(hash)
+            return unless hash.is_a?(Hash)
+
+            hash.each do |key, value|
+              next unless date_suffix_key?(key)
+              next unless value.is_a?(String) || value.is_a?(Numeric)
+
+              return value
+            end
+
+            suffix_date_value(hash[:attributes]) || suffix_date_value(hash[:data])
+          end
+
+          # @param key [Symbol, String] candidate published-at key
+          # @return [Boolean] whether the key ends in a date suffix (_at/_on/At/On)
+          def date_suffix_key?(key)
+            key.to_s.match?(DATE_KEY_SUFFIX)
+          end
+
           # rubocop:disable Metrics/MethodLength
           # @param entry [Hash] raw article entry candidate
           # @return [Array<String>, nil] normalized unique categories

data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb

@@ -6,11 +6,6 @@ module Html2rss
       class JsonState
         # Identifies arrays that look like collections of article hashes.
         module CandidateDetector
-          # Preferred keys when extracting title-like values from state payloads.
-          TITLE_KEYS = %i[title headline name text].freeze
-          # Preferred keys when extracting URL-like values from state payloads.
-          URL_KEYS = %i[url link href permalink slug path canonicalUrl shortUrl].freeze
-
           module_function
 
           # @param document [Hash, Array, Object] candidate document node
@@ -48,13 +43,14 @@ module Html2rss
           # @param object [Hash] article candidate object
           # @return [Object, nil] detected title-like value
           def title_from(object)
-            ValueFinder.fetch(object, TITLE_KEYS)
+            ValueFinder.fetch(object, ArticleNormalizer::TITLE_KEYS)
           end
 
           # @param object [Hash] article candidate object
-          # @return [Object, nil] detected URL-like value
+          # @return [Object, nil] detected URL-like value (including links.self)
           def url_from(object)
-            ValueFinder.fetch(object, URL_KEYS)
+            ValueFinder.fetch(object, ArticleNormalizer::URL_KEYS) ||
+              ValueFinder.fetch(object[:links], ArticleNormalizer::URL_KEYS)
           end
         end
       end

data/lib/html2rss/auto_source/scraper/json_state.rb

@@ -14,6 +14,11 @@ module Html2rss
         # @return [Symbol] scraper config key
         def self.options_key = :json_state
 
+        ##
+        # @param _opts [Hash] unused options
+        # @return [Integer]
+        def self.request_slots(_opts = {}) = 0
+
         class << self
           # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
           def articles?(parsed_body)

data/lib/html2rss/auto_source/scraper/meta_oembed.rb

@@ -8,7 +8,7 @@ module Html2rss
     module Scraper
       ##
       # Scrapes OpenGraph meta tags and oEmbed JSON descriptors from HTML pages.
-      class MetaOembed
+      class MetaOembed # rubocop:disable Metrics/ClassLength -- request_slots + extractable? stay on the scraper
         include Enumerable
 
         # Selector for OpenGraph meta tags.
@@ -30,6 +30,10 @@ module Html2rss
         # @return [Symbol] scraper config key
         def self.options_key = :meta_oembed
 
+        # @param _opts [Hash] unused options
+        # @return [Integer]
+        def self.request_slots(_opts = {}) = 0
+
         class << self
           # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
           # @return [Boolean] whether OpenGraph tags or oEmbed link exist
@@ -57,6 +61,9 @@ module Html2rss
           @request_session = request_session
         end
 
+        # @return [Boolean]
+        def extractable? = self.class.articles?(parsed_body)
+
         ##
         # Yields normalized article hash extracted from OpenGraph meta tags and optional oEmbed data.
         #