html2rss 0.28.0 → 0.29.1

data/lib/html2rss.rb

@@ -16,7 +16,7 @@ require 'html2rss/defaults'
 
 ##
 # The Html2rss namespace.
-module Html2rss
+module Html2rss # rubocop:disable Metrics/ModuleLength
   ##
   # The logger instance.
   module Log
@@ -28,6 +28,160 @@ module Html2rss
     end
   end
 
+  ##
+  # Cheap diagnostics for a URL (final URL, status, alternates, surface).
+  # Golden path step 1 (optional); use {.recon} for verdict and native_feed.
+  #
+  # @param url [String] source page URL
+  # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+  # @param deep [Boolean] when true and strategy is :auto, one Botasaurus hop if configured
+  # @return [Html2rss::PageRecon::Diagnostics::Report]
+  def self.inspect(url, strategy: :auto, deep: false, **)
+    PageRecon::Diagnostics.call(url:, strategy:, deep:, **)
+  end
+
+  ##
+  # Curation verdict and native_feed preference for a URL.
+  # Golden path step 2 (optional); adds verdict beyond {.inspect}.
+  #
+  # @param url [String, Html2rss::Url] source page URL
+  # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+  # @return [Html2rss::Recon::Result]
+  def self.recon(url, strategy: :auto, **)
+    Recon.call(url, strategy:, **)
+  end
+
+  ##
+  # Derives a reusable YAML-ready feed config from a URL.
+  # Golden path step 3.
+  #
+  # @param url [String] source page URL
+  # @param strategy [Symbol] request strategy (+:auto+, +:faraday+, +:botasaurus+)
+  # @option options [String, nil] :items_selector optional CSS selector hint for items
+  # @option options [Array<String>, nil] :topics optional directory topics override
+  # @option options [String, nil] :title optional title override
+  # @option options [String, nil] :summary optional summary override
+  # @option options [Boolean] :force whether to ignore native feed detection
+  # @option options [Boolean, nil] :enhance whether to force enhance: true
+  # @option options [Integer, nil] :max_redirects optional redirect limit override
+  # @option options [Integer, nil] :max_requests optional request budget override
+  # @option options [Integer, nil] :limit max articles to keep
+  # @option options [String, nil] :local_file_path optional local HTML file path
+  # @return [Html2rss::Capture::CaptureResult]
+  def self.capture(url, strategy: :auto, **)
+    Capture.build(url, strategy:, **)
+  end
+
+  ##
+  # Validates a config hash, YAML string, or file path against the schema.
+  # Side door: schema-only check without live extraction.
+  #
+  # @param config_input [Hash, String]
+  # @param feed_name [String, nil]
+  # @param params [Hash]
+  # @return [Dry::Validation::Result, Html2rss::Config::ValidationResult]
+  def self.validate(config_input, feed_name = nil, params: {})
+    _raw, validation = Config.resolve_and_validate(config_input, feed_name:, params:)
+    validation
+  end
+
+  ##
+  # Validates schema and asserts live item extraction.
+  # Golden path step 4.
+  #
+  # @param config_input [Hash, String] config hash, YAML string, or file path
+  # @param feed_name [String, nil] optional feed name in multi-feed file
+  # @param min_items [Integer] minimum required items (default: 1)
+  # @param params [Hash] dynamic feed params
+  # @param strategy [Symbol, nil] optional strategy override
+  # @param strict_quality [Boolean] when true, fail on ship-quality audit thresholds
+  # @param compare_enhance [Boolean] diagnostic enhance off vs on comparison on cached HTML
+  # @return [Html2rss::Test::Result]
+  def self.test(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, strict_quality: false, # rubocop:disable Metrics/ParameterLists
+                compare_enhance: false)
+    Test.call(config_input, feed_name, min_items:, params:, strategy:, strict_quality:, compare_enhance:)
+  end
+
+  ##
+  # Ships RSS from a validated config (user-facing verb for {.feed_result}).
+  # Golden path step 5.
+  #
+  # @param raw_config [Hash{Symbol => Object}] feed configuration
+  # @return [Html2rss::FeedResult]
+  def self.apply(raw_config)
+    feed_result(raw_config)
+  end
+
+  # rubocop:disable Metrics/ParameterLists
+
+  ##
+  # One-shot auto-source scrape from a URL (user-facing verb for {.auto_feed_result}).
+  #
+  # @param url [String] source page URL
+  # @param strategy [Symbol] request strategy to use
+  # @param items_selector [String, nil] optional selector hint for item extraction
+  # @param max_redirects [Integer, nil] optional redirect limit override
+  # @param max_requests [Integer] optional request budget override (default: 4 for sitemap sub-fetches)
+  # @param local_file_path [String, nil] optional local HTML file path
+  # @param limit [Integer, nil] max articles to keep (default: {AutoSource::DEFAULT_LIMIT})
+  # @return [Html2rss::FeedResult]
+  def self.scrape(url,
+                  strategy: :auto,
+                  items_selector: nil,
+                  max_redirects: nil,
+                  max_requests: 4,
+                  local_file_path: nil,
+                  limit: nil)
+    auto_feed_result(url, strategy:, items_selector:, max_redirects:, max_requests:,
+                          local_file_path:, limit:)
+  end
+
+  ##
+  # Scrapes multiple URLs in parallel using auto-source article discovery.
+  #
+  # @param urls [Enumerable<String>] list of URLs to scrape
+  # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+  # @param limit [Integer] max articles to keep per URL (default: 10)
+  # @param concurrency [Integer] max worker threads (default: 5)
+  # @return [Html2rss::Batch::BatchResult]
+  def self.batch_scrape(urls, strategy: :auto, limit: 10, concurrency: Batch::DEFAULT_CONCURRENCY)
+    Batch.batch_scrape(urls:, strategy:, limit:, concurrency:)
+  end
+
+  ##
+  # Inspects multiple URLs in parallel with per-URL error isolation.
+  #
+  # @param urls [Enumerable<String>] list of URLs to inspect
+  # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+  # @param concurrency [Integer] max worker threads (default: 5)
+  # @return [Html2rss::Batch::BatchResult]
+  def self.batch_inspect(urls, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY)
+    Batch.batch_inspect(urls:, strategy:, concurrency:)
+  end
+
+  ##
+  # Runs recon across multiple URLs in parallel with per-URL error isolation.
+  #
+  # @param urls [Enumerable<String>] list of URLs to recon
+  # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+  # @param concurrency [Integer] max worker threads (default: 5)
+  # @option options [String, nil] :cache_dir optional HTML cache directory
+  # @return [Html2rss::Batch::BatchResult]
+  def self.batch_recon(urls, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY, **)
+    Batch.batch_recon(urls:, strategy:, concurrency:, **)
+  end
+
+  # rubocop:enable Metrics/ParameterLists
+
+  ##
+  # Exports the configuration JSON Schema as JSON string.
+  #
+  # @param pretty [Boolean] whether to pretty-print JSON
+  # @return [String]
+  def self.schema_json(pretty: true)
+    Config.json_schema_json(pretty:)
+  end
+
   ##
   # Loads a feed configuration from YAML.
   #
@@ -143,39 +297,9 @@ module Html2rss
                           local_file_path:, limit:).to_json_feed
   end
 
-  ##
-  # Analyzes a URL and produces a reusable YAML-ready feed config hash.
-  #
-  # Uses auto-source discovery to extract articles, then derives CSS selectors
-  # from the structural analysis.
-  #
-  # @param url [String] source page URL
-  # @param strategy [Symbol] request strategy (+:auto+, +:faraday+, +:botasaurus+)
-  # @param items_selector [String, nil] optional CSS selector hint for items
-  # @param max_redirects [Integer, nil] optional redirect limit override
-  # @param max_requests [Integer, nil] optional request budget override
-  # @param limit [Integer, nil] max articles to keep
-  # @param local_file_path [String, nil] optional local HTML file path
-  # @return [Hash] feed config hash with +:channel+ and +:selectors+
-  def self.capture(url,
-                   strategy: :auto,
-                   items_selector: nil,
-                   max_redirects: nil,
-                   max_requests: nil,
-                   limit: nil,
-                   local_file_path: nil)
-    Capture.build(url,
-                  strategy:,
-                  items_selector:,
-                  max_redirects:,
-                  max_requests:,
-                  limit:,
-                  local_file_path:).config
-  end
-
   # rubocop:enable Metrics/ParameterLists
 
-  # rubocop:disable ThreadSafety/ClassInstanceVariable
+  # rubocop:disable-next ThreadSafety/ClassInstanceVariable
   class << self
     ##
     # @return [Html2rss::Defaults] the global defaults instance
@@ -217,12 +341,11 @@ module Html2rss
       logger.level = defaults.log_level if logger.respond_to?(:level=)
     end
   end
-  # rubocop:enable ThreadSafety/ClassInstanceVariable
 
   class << self
     private
 
-    # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength
+    # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength
     def build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
                                  local_file_path: nil, limit: nil)
       config = Config.auto_source_config(
@@ -237,7 +360,6 @@ module Html2rss
       end
       config
     end
-    # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
   end
 
   logger.level = defaults.log_level if logger.respond_to?(:level=)

data/schema/html2rss-config.schema.json

@@ -97,7 +97,12 @@
               "consumer",
               "civic",
               "product",
-              "research"
+              "research",
+              "health",
+              "culture",
+              "education",
+              "rights",
+              "transport"
             ]
           },
           "minItems": 1
@@ -443,7 +448,8 @@
               "type": "boolean",
               "not": {
                 "type": "null"
-              }
+              },
+              "description": "List-card enrichment: run Html::ArticleExtractor on each matched item node to fill missing fields from the card HTML."
             },
             "pagination": {
               "description": "Pagination configuration or maximum page count integer.",
@@ -504,7 +510,7 @@
           "required": [
             "selector"
           ],
-          "description": "Defines the items selector and optional enhancement settings."
+          "description": "Defines the items selector and list-card enhance settings."
         },
         "enclosure": {
           "$schema": "http://json-schema.org/draft-06/schema#",
@@ -546,9 +552,6 @@
                   {
                     "$ref": "#/$defs/post_processors/gsub"
                   },
-                  {
-                    "$ref": "#/$defs/post_processors/html_to_markdown"
-                  },
                   {
                     "$ref": "#/$defs/post_processors/markdown_to_html"
                   },
@@ -638,9 +641,6 @@
                   {
                     "$ref": "#/$defs/post_processors/gsub"
                   },
-                  {
-                    "$ref": "#/$defs/post_processors/html_to_markdown"
-                  },
                   {
                     "$ref": "#/$defs/post_processors/markdown_to_html"
                   },
@@ -875,26 +875,6 @@
         ],
         "additionalProperties": true
       },
-      "html_to_markdown": {
-        "type": "object",
-        "title": "html_to_markdown",
-        "description": "Sanitize HTML then convert it to Markdown (via ReverseMarkdown).",
-        "examples": [
-          {
-            "name": "html_to_markdown"
-          }
-        ],
-        "properties": {
-          "name": {
-            "type": "string",
-            "const": "html_to_markdown"
-          }
-        },
-        "required": [
-          "name"
-        ],
-        "additionalProperties": true
-      },
       "markdown_to_html": {
         "type": "object",
         "title": "markdown_to_html",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.28.0
+  version: 0.29.1
 platform: ruby
 authors:
 - Gil Desmarais
@@ -203,20 +203,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: reverse_markdown
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: rss
   requirement: !ruby/object:Gem::Requirement
@@ -351,10 +337,14 @@ files:
 - lib/html2rss/auto_source/segmenter/list.rb
 - lib/html2rss/auto_source/segmenter/primary_link.rb
 - lib/html2rss/auto_source/segmenter/semantic.rb
+- lib/html2rss/batch.rb
 - lib/html2rss/capture.rb
 - lib/html2rss/capture/README.md
 - lib/html2rss/channel.rb
 - lib/html2rss/cli.rb
+- lib/html2rss/cli/probe_view.rb
+- lib/html2rss/cli/render.rb
+- lib/html2rss/cli/validate.rb
 - lib/html2rss/config.rb
 - lib/html2rss/config/auto_source_contract.rb
 - lib/html2rss/config/dynamic_params.rb
@@ -365,6 +355,7 @@ files:
 - lib/html2rss/config/selectors_validator.rb
 - lib/html2rss/config/validator.rb
 - lib/html2rss/defaults.rb
+- lib/html2rss/doctor/botasaurus.rb
 - lib/html2rss/error.rb
 - lib/html2rss/feed_builder.rb
 - lib/html2rss/feed_builder/item_presentation.rb
@@ -405,6 +396,7 @@ files:
 - lib/html2rss/html/feed_link.rb
 - lib/html2rss/html/navigator.rb
 - lib/html2rss/html/navigator/text_extractor.rb
+- lib/html2rss/html/probe.rb
 - lib/html2rss/html/rendering.rb
 - lib/html2rss/html/rendering/audio_renderer.rb
 - lib/html2rss/html/rendering/description_builder.rb
@@ -420,12 +412,18 @@ files:
 - lib/html2rss/link_destination/path_classifier.rb
 - lib/html2rss/link_destination/text_classifier.rb
 - lib/html2rss/mcp.rb
+- lib/html2rss/mcp/README.md
 - lib/html2rss/mcp/config_argument.rb
 - lib/html2rss/mcp/contract.rb
-- lib/html2rss/mcp/inspect.rb
 - lib/html2rss/mcp/outcome.rb
+- lib/html2rss/mcp/outcome/playbook.rb
+- lib/html2rss/mcp/runtime.rb
 - lib/html2rss/mcp/server.rb
+- lib/html2rss/mcp/server/tools.rb
 - lib/html2rss/page_recon.rb
+- lib/html2rss/page_recon/README.md
+- lib/html2rss/page_recon/diagnostics.rb
+- lib/html2rss/recon.rb
 - lib/html2rss/request_service.rb
 - lib/html2rss/request_service/blocked_surface.rb
 - lib/html2rss/request_service/botasaurus_contract.rb
@@ -469,7 +467,6 @@ files:
 - lib/html2rss/selectors/post_processors.rb
 - lib/html2rss/selectors/post_processors/base.rb
 - lib/html2rss/selectors/post_processors/gsub.rb
-- lib/html2rss/selectors/post_processors/html_to_markdown.rb
 - lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb
 - lib/html2rss/selectors/post_processors/html_transformers/wrap_img_in_a.rb
 - lib/html2rss/selectors/post_processors/markdown_to_html.rb
@@ -494,6 +491,8 @@ files:
 - lib/html2rss/syndication/candidate_catalog.rb
 - lib/html2rss/syndication/discovery.rb
 - lib/html2rss/syndication/parser.rb
+- lib/html2rss/test.rb
+- lib/html2rss/test/enhance_audit.rb
 - lib/html2rss/url.rb
 - lib/html2rss/version.rb
 - lib/tasks/config_schema.rake
@@ -503,7 +502,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.28.0
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.29.1
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:

data/lib/html2rss/mcp/inspect.rb

@@ -1,138 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module MCP
-    ##
-    # Diagnostic inspect path (not Capture ownership). Fetches once, then delegates
-    # shared recon to {Html2rss::PageRecon}; adds MCP-only scraper/XHR diagnostics.
-    module Inspect
-      module_function
-
-      ##
-      # @param url [String]
-      # @param strategy [String, Symbol]
-      # @return [Hash]
-      def call(url:, strategy: :auto)
-        resolved = FeedPipeline::StrategyPlan.concrete_for_diagnostic(strategy)
-        response = fetch_response(url, resolved)
-        recon = PageRecon.call(response:, url:, strategy: resolved)
-
-        result = recon.to_h.merge(
-          strategy: resolved,
-          scraper_eligibility: scraper_info(safe_parsed_body(response))
-        )
-        result[:xhr_capture] = xhr_capture_info(response) if resolved == :botasaurus
-        result
-      end
-
-      ##
-      # @param response [Html2rss::RequestService::Response]
-      # @return [Hash] redacted XHR capture diagnostics (no query strings)
-      def xhr_capture_info(response)
-        captured = response.captured_responses
-        {
-          count: captured.size,
-          sample_endpoints: captured.first(5).filter_map { |entry| redacted_endpoint(entry) },
-          candidate_articles: captured.any? { |entry| xhr_candidate_articles?(entry) }
-        }
-      end
-      module_function :xhr_capture_info
-
-      ##
-      # @param entry [Hash] captured response hash
-      # @return [String, nil] scheme+host+path only
-      def redacted_endpoint(entry)
-        raw = entry['url'] || entry[:url]
-        return unless raw
-
-        uri = URI.parse(raw.to_s)
-        return unless uri.scheme && uri.host
-
-        "#{uri.scheme}://#{uri.host}#{uri.path}"
-      rescue URI::InvalidURIError
-        nil
-      end
-      module_function :redacted_endpoint
-
-      ##
-      # @param entry [Hash] captured response hash
-      # @return [Boolean]
-      def xhr_candidate_articles?(entry)
-        body = entry['body'] || entry[:body]
-        return false unless body.is_a?(String)
-
-        document = JSON.parse(body, symbolize_names: true)
-        AutoSource::Scraper::JsonState::CandidateDetector.candidate_array?(document)
-      rescue JSON::ParserError
-        false
-      end
-      module_function :xhr_candidate_articles?
-
-      ##
-      # @param url [String]
-      # @param strategy [Symbol]
-      # @return [Html2rss::RequestService::Response]
-      def fetch_response(url, strategy) # rubocop:disable Metrics/MethodLength -- session construction
-        raw_config = Config.auto_source_config(
-          url:,
-          request_controls: Config::RequestControls.from_shortcut(strategy:)
-        )
-        raw_config[:strategy] = strategy
-        config = Config.from_hash(raw_config)
-        resources = FeedPipeline::RuntimePolicy.resources_for(config)
-        session = RequestSession.build(
-          config:,
-          strategy: config.strategy,
-          budget: resources.budget,
-          policy: resources.policy
-        )
-        session.fetch_initial_response
-      end
-      module_function :fetch_response
-
-      ##
-      # @param parsed [Object] parsed response body
-      # @return [Array<String>, Hash]
-      def scraper_info(parsed)
-        return { error: 'Response is not HTML' } unless parsed.is_a?(Nokogiri::HTML::Document)
-
-        begin
-          Html2rss::AutoSource::Scraper.from(parsed).map(&:name)
-        rescue Html2rss::AutoSource::Scraper::NoScraperFound => error
-          { none_found: error.category.to_s }
-        end
-      end
-      module_function :scraper_info
-
-      ##
-      # @param response [Html2rss::RequestService::Response]
-      # @return [Hash, nil]
-      def sst_stats_from(response)
-        return nil unless response.html_response?
-
-        recon = PageRecon.call(response:, url: response.url)
-        return nil unless recon.sst
-
-        { node_count: recon.sst[:node_count], degraded: recon.sst[:degraded] }
-      end
-      module_function :sst_stats_from
-
-      ##
-      # @param sst [Html2rss::SST::Document]
-      # @param url [String]
-      # @return [Array]
-      def discover_segments(sst, url) = PageRecon.discover_segments(sst, url)
-      module_function :discover_segments
-
-      def safe_parsed_body(response)
-        return unless response.html_response?
-
-        response.parsed_body
-      rescue RequestService::UnsupportedResponseContentType
-        nil
-      end
-      module_function :safe_parsed_body
-      private_class_method :safe_parsed_body
-    end
-  end
-end

data/lib/html2rss/selectors/post_processors/html_to_markdown.rb

@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-require 'reverse_markdown'
-
-module Html2rss
-  class Selectors
-    module PostProcessors
-      ##
-      # Returns HTML code as Markdown formatted String.
-      # Before converting to markdown, the HTML is sanitized with SanitizeHtml.
-      # Imagine this HTML structure:
-      #
-      #     <section>
-      #       Lorem <b>ipsum</b> dolor...
-      #       <iframe src="https://evil.corp/miner"></iframe>
-      #       <script>alert();</script>
-      #     </section>
-      #
-      # YAML usage example:
-      #
-      #    selectors:
-      #      description:
-      #        selector: section
-      #        extractor: html
-      #        post_process:
-      #          name: html_to_markdown
-      #
-      # Would return:
-      #    'Lorem **ipsum** dolor'
-      class HtmlToMarkdown < Base
-        # JSON Schema description exported via +schema_doc+.
-        DESCRIPTION = 'Sanitize HTML then convert it to Markdown (via ReverseMarkdown).'
-
-        # Example post-process objects for JSON Schema +examples+.
-        EXAMPLES = [
-          { 'name' => 'html_to_markdown' }
-        ].freeze
-
-        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
-        def self.schema_doc = SchemaDoc.for_post_processor(name: :html_to_markdown, klass: self)
-
-        # @param value [String] extracted selector value
-        # @param context [Selectors::Context] post-processor context
-        # @return [void]
-        def self.validate_args!(value, context)
-          assert_type value, String, :value, context:
-        end
-
-        ##
-        # @return [String] formatted in Markdown
-        def get
-          sanitized_value = SanitizeHtml.new(value, context).get
-
-          ReverseMarkdown.convert(sanitized_value)
-        end
-      end
-    end
-  end
-end