html2rss 0.27.2 → 0.29.1

data/lib/html2rss/auto_source/scraper/schema.rb

@@ -15,27 +15,33 @@ module Html2rss
       class Schema
         include Enumerable
 
-        # Selector for JSON-LD script tags containing Schema.org objects.
-        TAG_SELECTOR = 'script[type="application/ld+json"]'
+        # Matches a leading schema.org URL prefix on @type values (http or https).
+        SCHEMA_ORG_PREFIX_RE = %r{\Ahttps?://schema\.org/}i
+
+        # Container types that must never be emitted as feed items (walk children only).
+        DENIED_CONTAINER_TYPES = Set[
+          'ItemList', 'Blog', 'BreadcrumbList', 'WebPage', 'CollectionPage'
+        ].freeze
+
+        # Canonical Schema.org type names keyed by folded wire forms (case-insensitive lookup).
+        CANONICAL_BY_DOWNCASE = begin
+          canonical_types = Thing::SUPPORTED_TYPES | ItemList::SUPPORTED_TYPES | DENIED_CONTAINER_TYPES | Set['Product']
+          canonical_types.to_h { |type| [::Html2rss::Html::Probe.fold(type), type] }.freeze
+        end.freeze
 
         # Pre-compiled regex for supported schema types (short name or schema.org URL; string or array @type).
         # Allows preceding entries in a JSON @type array (e.g. ["WebPage","NewsArticle"]).
         SUPPORTED_TYPES_RE = begin
           types = Thing::SUPPORTED_TYPES | ItemList::SUPPORTED_TYPES
           type_re = Regexp.union(types.to_a)
-          %r{"@type"\s*:\s*(?:\[\s*(?:"[^"]*"\s*,\s*)*)?"(?:https?://schema\.org/)?(?:#{type_re.source})"}
+          %r{(?i)"@type"\s*:\s*(?:\[\s*(?:"[^"]*"\s*,\s*)*)?"(?:https?://schema\.org/)?(?:#{type_re.source})"}
         end.freeze
 
-        # Matches a leading schema.org URL prefix on @type values (http or https).
-        SCHEMA_ORG_PREFIX_RE = %r{\Ahttps?://schema\.org/}i
-
         # Prefer these keys when recursively walking unsupported container objects.
         COLLECTION_KEYS = %i[itemListElement blogPost mainEntity hasPart].freeze
 
-        # Container types that must never be emitted as feed items (walk children only).
-        DENIED_CONTAINER_TYPES = Set[
-          'ItemList', 'Blog', 'BreadcrumbList', 'WebPage', 'CollectionPage'
-        ].freeze
+        # Shared empty type set for nil/unsupported wire forms (avoids per-call Set.new).
+        EMPTY_TYPES = Set.new.freeze
 
         # @return [Symbol] scraper config key
         def self.options_key = :schema
@@ -44,7 +50,8 @@ module Html2rss
           # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
           # @return [Boolean] whether the page includes supported schema types
           def articles?(parsed_body)
-            parsed_body.css(TAG_SELECTOR).any? { |script| supported_schema_type?(script) }
+            ::Html2rss::Html::Probe.scripts(parsed_body, ::Html2rss::Html::Probe::APPLICATION_LD_JSON)
+                                   .any? { |script| supported_schema_type?(script) }
           end
 
           # @param script [Nokogiri::XML::Element] schema JSON-LD script tag
@@ -105,14 +112,26 @@ module Html2rss
             case object
             when Array
               object.each_with_object(Set.new) { |item, set| set.merge(normalize_types(item)) }
-            when String, Symbol
-              short = object.to_s.sub(SCHEMA_ORG_PREFIX_RE, '')
-              short.empty? ? Set.new : Set[short]
             else
-              Set.new
+              name = canonicalize_type(object)
+              name ? Set[name] : EMPTY_TYPES
             end
           end
 
+          # Canonical short Schema.org type name for a scalar wire form.
+          #
+          # @param object [String, Symbol, nil] raw `@type` / itemtype token
+          # @return [String, nil]
+          # @api private
+          def canonicalize_type(object)
+            return unless object.is_a?(String) || object.is_a?(Symbol)
+
+            short = object.to_s.sub(SCHEMA_ORG_PREFIX_RE, '')
+            return if short.empty?
+
+            CANONICAL_BY_DOWNCASE.fetch(::Html2rss::Html::Probe.fold(short), short)
+          end
+
           private
 
           # @param hash [Hash] candidate schema object
@@ -150,15 +169,15 @@ module Html2rss
         ##
         # @yield [Hash] Each scraped article_hash
         # @return [Array<Hash>] the scraped article_hashes
-        def each(&)
+        def each
           return enum_for(:each) unless block_given?
 
-          schema_objects.filter_map do |schema_object|
+          schema_objects.each do |schema_object|
             next unless (klass = self.class.scraper_for_schema_object(schema_object))
             next unless (results = klass.new(schema_object, url:).call)
 
             if results.is_a?(Array)
-              results.each { |result| yield(result) } # rubocop:disable Style/ExplicitBlockArgument
+              results.each { yield(_1) }
             else
               yield(results)
             end
@@ -168,9 +187,8 @@ module Html2rss
         private
 
         def schema_objects
-          @parsed_body.css(TAG_SELECTOR).flat_map do |tag|
-            Schema.from(tag)
-          end
+          ::Html2rss::Html::Probe.scripts(@parsed_body, ::Html2rss::Html::Probe::APPLICATION_LD_JSON)
+                                 .flat_map { |tag| Schema.from(tag) }
         end
 
         attr_reader :parsed_body, :url

data/lib/html2rss/auto_source/scraper.rb

@@ -23,7 +23,9 @@ module Html2rss
 
       # Extraction tiers: merge within a tier, then stop when AutoSource has enough articles.
       # Heuristic scrapers are separate tiers so SemanticHtml can satisfy before Html runs.
+      # NativeFeed is tier 0 so syndication wins before structured HTML scrapers.
       SCRAPER_TIERS = [
+        [NativeFeed].freeze,
         [Schema, Microdata, Microformats2, JsonState, XhrArticles].freeze,
         [WordpressApi, Sitemap, MetaOembed].freeze,
         [SemanticHtml].freeze,
@@ -36,7 +38,7 @@ module Html2rss
       # Heuristic scrapers that share one memoized SST::Document per page.
       HEURISTIC_SCRAPERS = [SemanticHtml, Html].freeze
       # Scrapers that accept a shared follow-up +request_session+.
-      REQUEST_SESSION_SCRAPERS = [WordpressApi, Sitemap, MetaOembed].freeze
+      REQUEST_SESSION_SCRAPERS = [NativeFeed, WordpressApi, Sitemap, MetaOembed].freeze
       # Scrapers that consume browser-captured XHR/fetch JSON bodies.
       CAPTURED_RESPONSE_SCRAPERS = [XhrArticles].freeze
 
@@ -83,6 +85,7 @@ module Html2rss
       # Returns an array of scraper classes that claim to find articles in the parsed body.
       # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML document.
       # @param opts [Hash] The options hash.
+      # @option opts [Hash] :native_feed scraper toggle and configuration
       # @option opts [Hash] :wordpress_api scraper toggle and configuration
       # @option opts [Hash] :schema scraper toggle and configuration
       # @option opts [Hash] :microdata scraper toggle and configuration
@@ -115,7 +118,7 @@ module Html2rss
       # @return [SST::Document, nil]
       def self.normalize_sst(parsed_body)
         SST::Normalizer.call(parsed_body)
-      rescue ArgumentError
+      rescue SST::Normalizer::EmptyTree
         nil
       end
 
@@ -142,7 +145,7 @@ module Html2rss
       # @option opts [Hash] :html scraper toggle and configuration
       # @option opts [Hash] :sitemap scraper toggle and configuration
       # @return [Object, nil]
-      # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength -- construction context for structured and heuristic scrapers
+      # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- construction context for structured and heuristic scrapers
       def self.build_instance(scraper, parsed_body, opts:, url:, request_session: nil, body: nil, document: nil,
                               link_resolver: nil, captured_responses: [])
         return unless opts.dig(scraper.options_key, :enabled)
@@ -159,7 +162,6 @@ module Html2rss
         )
         scraper.new(parsed_body, url:, **kwargs, **scraper_opts)
       end
-      # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
 
       ##
       # @param instance [Object]

data/lib/html2rss/auto_source/segmenter/cluster.rb

@@ -64,7 +64,7 @@ module Html2rss
         def normalize_class(class_names)
           return '' if class_names.empty?
 
-          class_names.sort.join(' ')
+          class_names.map { ::Html2rss::Html::Probe.fold(_1) }.sort.join(' ')
         end
         module_function :normalize_class
         private_class_method :normalize_class
@@ -122,7 +122,7 @@ module Html2rss
             groups.sort_by { |_key, nodes| -nodes.size }.first(MAX_GROUPS).to_h
           end
 
-          # rubocop:disable Metrics/MethodLength
+          # rubocop:disable-next Metrics/MethodLength
           def container_of?(nodes_a, nodes_b)
             return false unless @layout_tags.include?(nodes_b.first.name)
 
@@ -139,7 +139,6 @@ module Html2rss
               count > 1
             end
           end
-          # rubocop:enable Metrics/MethodLength
 
           def resolve_1_to_1_overlap(cls_a, cls_b, groups, discarded)
             nodes_a = groups[cls_a]

data/lib/html2rss/auto_source/segmenter/primary_link.rb

@@ -37,7 +37,7 @@ module Html2rss
 
         private
 
-        # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
         def candidate_facts(anchor, container)
           destination = @link_resolver.destination_facts(anchor)
           return unless destination
@@ -63,7 +63,6 @@ module Html2rss
 
           { anchor:, destination: destination.destination, score: }
         end
-        # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
 
         def first_heading(container)
           (@headings ||= {}.compare_by_identity)[container] ||= container.find(&:heading?)

data/lib/html2rss/auto_source/segmenter.rb

@@ -19,11 +19,10 @@ module Html2rss
       # @option opts [Integer] :use_top_selectors list strategy selector budget
       # @option opts [Scoring::LinkResolver, nil] :link_resolver page-scoped destination facts cache
       # @return [Array<Segment>]
-      # rubocop:disable Style/ArgumentsForwarding -- keep named opts for YARD @option
+      # rubocop:disable-next Style/ArgumentsForwarding -- keep named opts for YARD @option
       def self.call(document, base_url:, strategy:, **opts)
         new(document, base_url:, strategy:, **opts).call
       end
-      # rubocop:enable Style/ArgumentsForwarding
 
       # @param document [SST::Document]
       # @param base_url [String, Html2rss::Url]

data/lib/html2rss/auto_source.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require 'dry-validation'
+require 'json'
+require 'nokogiri'
 
 module Html2rss
   ##
@@ -20,10 +22,25 @@ module Html2rss
     # Default max articles to keep (also the short-circuit floor across scraper tiers).
     DEFAULT_LIMIT = 25
 
+    # Operational scraper failures quarantined within a tier; programmer errors propagate.
+    OPERATIONAL_ERRORS = [
+      Html2rss::Error,
+      ArgumentError,
+      ::JSON::ParserError,
+      ::Nokogiri::SyntaxError
+    ].freeze
+
     # Default auto-source configuration shipped for scraper and cleanup behavior.
     DEFAULT_CONFIG = {
       limit: DEFAULT_LIMIT,
+      entry_resolution: {
+        enabled: true,
+        max_probes: 5
+      }.freeze,
       scraper: {
+        native_feed: {
+          enabled: true
+        },
         wordpress_api: {
           enabled: true
         },
@@ -131,7 +148,7 @@ module Html2rss
 
     attr_reader :url, :parsed_body, :body, :request_session, :captured_responses
 
-    # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+    # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
     def extract_articles
       articles = []
       matched = false
@@ -176,7 +193,6 @@ module Html2rss
       @admission_drops = result.drop_tallies
       result.articles.first(article_limit)
     end
-    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
 
     def enough_articles?(articles)
       return false if articles.size < article_limit
@@ -204,15 +220,20 @@ module Html2rss
     end
 
     def run_scraper(instance)
-      instance.each.map do |item|
-        case item
-        when Article
-          item
-        when Hash
-          Article.new(**item, scraper: instance.class)
-        else
-          raise TypeError, "#{instance.class} yielded #{item.class}; expected Article or Hash"
-        end
+      instance.map { |item| article_from_scraper_item(item, instance) }
+    rescue *OPERATIONAL_ERRORS => error
+      Log.warn("#{self.class}: #{instance.class} quarantined #{error.class}: #{error.message}")
+      []
+    end
+
+    def article_from_scraper_item(item, instance)
+      case item
+      when Article
+        item
+      when Hash
+        Article.new(**item, scraper: instance.class)
+      else
+        raise TypeError, "#{instance.class} yielded #{item.class}; expected Article or Hash"
       end
     end
 

data/lib/html2rss/batch.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Concurrency-bounded batch execution over enumerables with input order preservation
+  # and per-item error isolation.
+  module Batch
+    # Maximum permitted worker threads for batch operations.
+    MAX_CONCURRENCY = 10
+    # Default worker threads for batch operations.
+    DEFAULT_CONCURRENCY = 5
+
+    # Value object representing the outcome of a batch operation.
+    BatchResult = Data.define(:total, :successful, :results) do
+      ##
+      # @return [Hash{Symbol => Object}]
+      def to_h
+        { total:, successful:, results: }
+      end
+    end
+
+    class << self
+      ##
+      # Executes a block concurrently across items using a bounded worker pool.
+      # Results are returned in input order.
+      #
+      # @param items [Enumerable<Object>]
+      # @param concurrency [Integer] max parallel worker threads (1..10, default: 5)
+      # @yieldparam item [Object]
+      # @return [Array<Object>]
+      def map(items, concurrency: DEFAULT_CONCURRENCY, &)
+        list = Array(items)
+        return [] if list.empty?
+
+        worker_count = worker_pool_size(list.size, concurrency)
+        process_queue(list, worker_count, &)
+      end
+
+      ##
+      # Runs a batch operation that yields items and wraps the results in a {BatchResult}.
+      #
+      # @param items [Enumerable<Object>]
+      # @param concurrency [Integer] max parallel worker threads (1..10, default: 5)
+      # @yieldparam item [Object]
+      # @return [BatchResult]
+      def run(items, concurrency: DEFAULT_CONCURRENCY, &)
+        list = Array(items)
+        results = map(list, concurrency:, &)
+        successful = results.count { |entry| entry.is_a?(Hash) ? entry[:ok] : !entry.nil? }
+
+        BatchResult.new(total: list.size, successful:, results:)
+      end
+
+      ##
+      # Single-URL scrape wire extraction: auto_feed_result → JSON Feed items.
+      #
+      # @param url [String]
+      # @param strategy [Symbol, String]
+      # @param limit [Integer]
+      # @param items_selector [String, nil]
+      # @return [Hash{Symbol => Object}] +:strategy+, +:items+, +:channel_title+, +:admission_drops+
+      def scrape_wire(url:, strategy:, limit:, items_selector: nil) # rubocop:disable Metrics/MethodLength
+        plan = (strategy || :auto).to_sym
+        feed_args = { strategy: plan, limit: }
+        feed_args[:items_selector] = items_selector unless items_selector.nil?
+        feed_result = Html2rss.auto_feed_result(url, **feed_args)
+        feed = feed_result.to_json_feed
+        {
+          strategy: plan,
+          items: feed[:items] || [],
+          channel_title: feed[:title],
+          admission_drops: feed_result.status.admission_drops
+        }
+      end
+
+      ##
+      # Scrapes multiple URLs in parallel with per-URL error isolation.
+      #
+      # @param urls [Enumerable<String>] list of URLs to scrape
+      # @param strategy [Symbol, String] request strategy (+:auto+, +:faraday+, +:botasaurus+)
+      # @param limit [Integer] max articles to extract per URL
+      # @param concurrency [Integer] number of worker threads (default 5, max 10)
+      # @return [BatchResult]
+      def batch_scrape(urls:, strategy: :auto, limit: 10, concurrency: DEFAULT_CONCURRENCY)
+        run(urls, concurrency:) { |url| scrape_single_url(url:, strategy:, limit:) }
+      end
+
+      ##
+      # Inspects multiple URLs in parallel with per-URL error isolation.
+      #
+      # @param urls [Enumerable<String>] list of URLs to inspect
+      # @param strategy [Symbol, String] request strategy (+:auto+, +:faraday+, +:botasaurus+)
+      # @param concurrency [Integer] number of worker threads (default 5, max 10)
+      # @return [BatchResult]
+      def batch_inspect(urls:, strategy: :auto, concurrency: DEFAULT_CONCURRENCY)
+        reports = PageRecon::Diagnostics.batch(urls:, strategy:, concurrency:)
+        results = reports.map(&:to_wire_h)
+        successful = reports.count { |report| diagnostic_success?(report) }
+
+        BatchResult.new(total: results.size, successful:, results:)
+      end
+
+      ##
+      # Runs recon across multiple URLs in parallel with per-URL error isolation.
+      #
+      # @param urls [Enumerable<String>] list of URLs to recon
+      # @param strategy [Symbol, String] request strategy (+:auto+, +:faraday+, +:botasaurus+)
+      # @param concurrency [Integer] number of worker threads (default 5, max 10)
+      # @option options [String, nil] :cache_dir optional HTML cache directory
+      # @return [BatchResult]
+      def batch_recon(urls:, strategy: :auto, concurrency: DEFAULT_CONCURRENCY, **)
+        results = Recon.batch(urls, strategy: (strategy || :auto).to_sym, max_threads: concurrency, **)
+        successful = results.count { |r| r.status ? r.status < 400 : false }
+        BatchResult.new(total: results.size, successful:, results: results.map(&:to_h))
+      end
+
+      private
+
+      def diagnostic_success?(report)
+        status = report.to_wire_h[:status]
+        status ? status < 400 : false
+      end
+
+      def scrape_single_url(url:, strategy:, limit:)
+        wire = scrape_wire(url:, strategy:, limit:)
+        {
+          url: url.to_s, ok: true, items_count: wire[:items].size,
+          items: wire[:items], channel_title: wire[:channel_title]
+        }
+      rescue StandardError => error
+        { url: url.to_s, ok: false, error: error.message }
+      end
+
+      def worker_pool_size(item_count, requested_concurrency)
+        return 1 if item_count <= 1
+
+        [[1, requested_concurrency.to_i].max, item_count, MAX_CONCURRENCY].min
+      end
+
+      def process_queue(items, worker_count, &)
+        queue = build_work_queue(items)
+        results = Array.new(items.size)
+
+        threads = Array.new(worker_count) do
+          Thread.new { drain_queue(queue, results, &) } # rubocop:disable ThreadSafety/NewThread
+        end
+        threads.each(&:join)
+        results
+      end
+
+      def build_work_queue(items)
+        Queue.new.tap do |queue|
+          items.each_with_index { |item, index| queue << [item, index] }
+        end
+      end
+
+      def drain_queue(queue, results)
+        until queue.empty?
+          begin
+            item, index = queue.pop(true)
+          rescue ThreadError
+            break
+          end
+          results[index] = yield(item)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/capture/README.md

@@ -1,6 +1,10 @@
 # Capture
 
-`Html2rss.capture` (and CLI `html2rss capture`) analyzes a URL through the feed pipeline and produces a reusable config with **items selector + `enhance: true` only** — no title/url/description attribute-selector soup. At feed-build time, `enhance: true` fills missing article fields via `Html::ArticleExtractor` on each matched item.
+`Html2rss.capture` (and CLI `html2rss capture`) analyzes a URL through the feed pipeline and produces a reusable config with **items selector + `enhance: true` only** — no title/url/description attribute-selector soup. At feed-build time, `enhance: true` fills missing article fields via `Html::ArticleExtractor` on each **matched list-card item node**.
+
+## API
+
+`Html2rss.capture` / `Capture.build` return `Capture::CaptureResult` — use `#yaml` or `#config`, not a bare Hash.
 
 ## When to use it
 
@@ -9,8 +13,9 @@ Point `capture` at a listing URL when you want a first-draft YAML config instead
 ## Gem API
 
 ```ruby
-config = Html2rss.capture('https://example.com/articles')
+result = Html2rss.capture('https://example.com/articles')
 
+# result.config =>
 # {
 #   channel: { url: "...", title: "...", time_zone: "UTC" },
 #   selectors: {
@@ -18,11 +23,13 @@ config = Html2rss.capture('https://example.com/articles')
 #   }
 # }
 
-File.write('my-feed.yml', YAML.dump(Html2rss::HashUtil.deep_stringify_keys(config)))
-feed = Html2rss.feed(config)
+File.write('my-feed.yml', result.yaml)
+rss = Html2rss.apply(result.config)
 ```
 
-`Capture.build` returns a `CaptureResult` with quality meta (`has_selectors`, `segment_strategy`, `admission_drops`, `selected_strategy`). `Html2rss.capture` returns only the config hash.
+`Html2rss.capture` and `Capture.build` both return a `CaptureResult` with YAML (`#yaml`, includes the schema modeline) and quality meta (`has_selectors`, `segment_strategy`, `admission_drops`, `selected_strategy`, `native_feed`).
+
+**Wire vs internal:** user-facing **`Html2rss.apply`** ships RSS from a config Hash. Pipeline internals **`Html2rss.feed` / `feed_result`** stay unchanged — use `apply` in CLI/MCP/docs examples, not `feed`.
 
 ### Options
 
@@ -30,15 +37,17 @@ feed = Html2rss.feed(config)
 Html2rss.capture('https://spa-site.com', strategy: :botasaurus)
 Html2rss.capture('https://example.com', items_selector: '.article-card')
 Html2rss.capture('https://example.com', strategy: :local_file, local_file_path: './page.html')
+Html2rss.capture('https://example.com', max_redirects: 8, max_requests: 4)
 ```
 
-`strategy: :auto` uses the same AutoFallback chain as scrape (`faraday` → `botasaurus`). When AutoFallback selects a concrete strategy (or you pin one), Capture **stamps** `strategy:` into the emitted config so later `Html2rss.feed(config)` replays the same transport.
+`strategy: :auto` uses the same AutoFallback chain as scrape (`faraday` → `botasaurus`). When AutoFallback selects a concrete strategy (or you pin one), Capture **stamps** `strategy:` into the emitted config so later `Html2rss.apply(config)` replays the same transport.
 
 ## CLI
 
 ```bash
 html2rss capture https://example.com/articles
 html2rss capture https://example.com --strategy botasaurus
+html2rss capture https://example.com --max-redirects 8 --max-requests 4
 html2rss capture https://example.com/articles > my-feed.yml
 html2rss capture https://example.com --input ./page.html
 html2rss capture https://example.com --explain   # quality JSON on stderr; YAML on stdout