html2rss 0.27.2 → 0.29.1

data/lib/html2rss.rb

@@ -16,7 +16,7 @@ require 'html2rss/defaults'
 
 ##
 # The Html2rss namespace.
-module Html2rss
+module Html2rss # rubocop:disable Metrics/ModuleLength
   ##
   # The logger instance.
   module Log
@@ -28,6 +28,160 @@ module Html2rss
     end
   end
 
+  ##
+  # Cheap diagnostics for a URL (final URL, status, alternates, surface).
+  # Golden path step 1 (optional); use {.recon} for verdict and native_feed.
+  #
+  # @param url [String] source page URL
+  # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+  # @param deep [Boolean] when true and strategy is :auto, one Botasaurus hop if configured
+  # @return [Html2rss::PageRecon::Diagnostics::Report]
+  def self.inspect(url, strategy: :auto, deep: false, **)
+    PageRecon::Diagnostics.call(url:, strategy:, deep:, **)
+  end
+
+  ##
+  # Curation verdict and native_feed preference for a URL.
+  # Golden path step 2 (optional); adds verdict beyond {.inspect}.
+  #
+  # @param url [String, Html2rss::Url] source page URL
+  # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+  # @return [Html2rss::Recon::Result]
+  def self.recon(url, strategy: :auto, **)
+    Recon.call(url, strategy:, **)
+  end
+
+  ##
+  # Derives a reusable YAML-ready feed config from a URL.
+  # Golden path step 3.
+  #
+  # @param url [String] source page URL
+  # @param strategy [Symbol] request strategy (+:auto+, +:faraday+, +:botasaurus+)
+  # @option options [String, nil] :items_selector optional CSS selector hint for items
+  # @option options [Array<String>, nil] :topics optional directory topics override
+  # @option options [String, nil] :title optional title override
+  # @option options [String, nil] :summary optional summary override
+  # @option options [Boolean] :force whether to ignore native feed detection
+  # @option options [Boolean, nil] :enhance whether to force enhance: true
+  # @option options [Integer, nil] :max_redirects optional redirect limit override
+  # @option options [Integer, nil] :max_requests optional request budget override
+  # @option options [Integer, nil] :limit max articles to keep
+  # @option options [String, nil] :local_file_path optional local HTML file path
+  # @return [Html2rss::Capture::CaptureResult]
+  def self.capture(url, strategy: :auto, **)
+    Capture.build(url, strategy:, **)
+  end
+
+  ##
+  # Validates a config hash, YAML string, or file path against the schema.
+  # Side door: schema-only check without live extraction.
+  #
+  # @param config_input [Hash, String]
+  # @param feed_name [String, nil]
+  # @param params [Hash]
+  # @return [Dry::Validation::Result, Html2rss::Config::ValidationResult]
+  def self.validate(config_input, feed_name = nil, params: {})
+    _raw, validation = Config.resolve_and_validate(config_input, feed_name:, params:)
+    validation
+  end
+
+  ##
+  # Validates schema and asserts live item extraction.
+  # Golden path step 4.
+  #
+  # @param config_input [Hash, String] config hash, YAML string, or file path
+  # @param feed_name [String, nil] optional feed name in multi-feed file
+  # @param min_items [Integer] minimum required items (default: 1)
+  # @param params [Hash] dynamic feed params
+  # @param strategy [Symbol, nil] optional strategy override
+  # @param strict_quality [Boolean] when true, fail on ship-quality audit thresholds
+  # @param compare_enhance [Boolean] diagnostic enhance off vs on comparison on cached HTML
+  # @return [Html2rss::Test::Result]
+  def self.test(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, strict_quality: false, # rubocop:disable Metrics/ParameterLists
+                compare_enhance: false)
+    Test.call(config_input, feed_name, min_items:, params:, strategy:, strict_quality:, compare_enhance:)
+  end
+
+  ##
+  # Ships RSS from a validated config (user-facing verb for {.feed_result}).
+  # Golden path step 5.
+  #
+  # @param raw_config [Hash{Symbol => Object}] feed configuration
+  # @return [Html2rss::FeedResult]
+  def self.apply(raw_config)
+    feed_result(raw_config)
+  end
+
+  # rubocop:disable Metrics/ParameterLists
+
+  ##
+  # One-shot auto-source scrape from a URL (user-facing verb for {.auto_feed_result}).
+  #
+  # @param url [String] source page URL
+  # @param strategy [Symbol] request strategy to use
+  # @param items_selector [String, nil] optional selector hint for item extraction
+  # @param max_redirects [Integer, nil] optional redirect limit override
+  # @param max_requests [Integer] optional request budget override (default: 4 for sitemap sub-fetches)
+  # @param local_file_path [String, nil] optional local HTML file path
+  # @param limit [Integer, nil] max articles to keep (default: {AutoSource::DEFAULT_LIMIT})
+  # @return [Html2rss::FeedResult]
+  def self.scrape(url,
+                  strategy: :auto,
+                  items_selector: nil,
+                  max_redirects: nil,
+                  max_requests: 4,
+                  local_file_path: nil,
+                  limit: nil)
+    auto_feed_result(url, strategy:, items_selector:, max_redirects:, max_requests:,
+                          local_file_path:, limit:)
+  end
+
+  ##
+  # Scrapes multiple URLs in parallel using auto-source article discovery.
+  #
+  # @param urls [Enumerable<String>] list of URLs to scrape
+  # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+  # @param limit [Integer] max articles to keep per URL (default: 10)
+  # @param concurrency [Integer] max worker threads (default: 5)
+  # @return [Html2rss::Batch::BatchResult]
+  def self.batch_scrape(urls, strategy: :auto, limit: 10, concurrency: Batch::DEFAULT_CONCURRENCY)
+    Batch.batch_scrape(urls:, strategy:, limit:, concurrency:)
+  end
+
+  ##
+  # Inspects multiple URLs in parallel with per-URL error isolation.
+  #
+  # @param urls [Enumerable<String>] list of URLs to inspect
+  # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+  # @param concurrency [Integer] max worker threads (default: 5)
+  # @return [Html2rss::Batch::BatchResult]
+  def self.batch_inspect(urls, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY)
+    Batch.batch_inspect(urls:, strategy:, concurrency:)
+  end
+
+  ##
+  # Runs recon across multiple URLs in parallel with per-URL error isolation.
+  #
+  # @param urls [Enumerable<String>] list of URLs to recon
+  # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+  # @param concurrency [Integer] max worker threads (default: 5)
+  # @option options [String, nil] :cache_dir optional HTML cache directory
+  # @return [Html2rss::Batch::BatchResult]
+  def self.batch_recon(urls, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY, **)
+    Batch.batch_recon(urls:, strategy:, concurrency:, **)
+  end
+
+  # rubocop:enable Metrics/ParameterLists
+
+  ##
+  # Exports the configuration JSON Schema as JSON string.
+  #
+  # @param pretty [Boolean] whether to pretty-print JSON
+  # @return [String]
+  def self.schema_json(pretty: true)
+    Config.json_schema_json(pretty:)
+  end
+
   ##
   # Loads a feed configuration from YAML.
   #
@@ -143,39 +297,9 @@ module Html2rss
                           local_file_path:, limit:).to_json_feed
   end
 
-  ##
-  # Analyzes a URL and produces a reusable YAML-ready feed config hash.
-  #
-  # Uses auto-source discovery to extract articles, then derives CSS selectors
-  # from the structural analysis.
-  #
-  # @param url [String] source page URL
-  # @param strategy [Symbol] request strategy (+:auto+, +:faraday+, +:botasaurus+)
-  # @param items_selector [String, nil] optional CSS selector hint for items
-  # @param max_redirects [Integer, nil] optional redirect limit override
-  # @param max_requests [Integer, nil] optional request budget override
-  # @param limit [Integer, nil] max articles to keep
-  # @param local_file_path [String, nil] optional local HTML file path
-  # @return [Hash] feed config hash with +:channel+ and +:selectors+
-  def self.capture(url,
-                   strategy: :auto,
-                   items_selector: nil,
-                   max_redirects: nil,
-                   max_requests: nil,
-                   limit: nil,
-                   local_file_path: nil)
-    Capture.build(url,
-                  strategy:,
-                  items_selector:,
-                  max_redirects:,
-                  max_requests:,
-                  limit:,
-                  local_file_path:).config
-  end
-
   # rubocop:enable Metrics/ParameterLists
 
-  # rubocop:disable ThreadSafety/ClassInstanceVariable
+  # rubocop:disable-next ThreadSafety/ClassInstanceVariable
   class << self
     ##
     # @return [Html2rss::Defaults] the global defaults instance
@@ -217,12 +341,11 @@ module Html2rss
       logger.level = defaults.log_level if logger.respond_to?(:level=)
     end
   end
-  # rubocop:enable ThreadSafety/ClassInstanceVariable
 
   class << self
     private
 
-    # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength
+    # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength
     def build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
                                  local_file_path: nil, limit: nil)
       config = Config.auto_source_config(
@@ -237,7 +360,6 @@ module Html2rss
       end
       config
     end
-    # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
   end
 
   logger.level = defaults.log_level if logger.respond_to?(:level=)

data/schema/html2rss-config.schema.json

@@ -97,7 +97,12 @@
               "consumer",
               "civic",
               "product",
-              "research"
+              "research",
+              "health",
+              "culture",
+              "education",
+              "rights",
+              "transport"
             ]
           },
           "minItems": 1
@@ -157,9 +162,40 @@
           },
           "exclusiveMinimum": 0
         },
+        "entry_resolution": {
+          "type": "object",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "not": {
+                "type": "null"
+              }
+            },
+            "max_probes": {
+              "type": "integer",
+              "not": {
+                "type": "null"
+              },
+              "exclusiveMinimum": 0
+            }
+          },
+          "required": []
+        },
         "scraper": {
           "type": "object",
           "properties": {
+            "native_feed": {
+              "type": "object",
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
+                }
+              },
+              "required": []
+            },
             "wordpress_api": {
               "type": "object",
               "properties": {
@@ -338,7 +374,14 @@
       "required": [],
       "default": {
         "limit": 25,
+        "entry_resolution": {
+          "enabled": true,
+          "max_probes": 5
+        },
         "scraper": {
+          "native_feed": {
+            "enabled": true
+          },
           "wordpress_api": {
             "enabled": true
           },
@@ -405,7 +448,8 @@
               "type": "boolean",
               "not": {
                 "type": "null"
-              }
+              },
+              "description": "List-card enrichment: run Html::ArticleExtractor on each matched item node to fill missing fields from the card HTML."
             },
             "pagination": {
               "description": "Pagination configuration or maximum page count integer.",
@@ -466,7 +510,7 @@
           "required": [
             "selector"
           ],
-          "description": "Defines the items selector and optional enhancement settings."
+          "description": "Defines the items selector and list-card enhance settings."
         },
         "enclosure": {
           "$schema": "http://json-schema.org/draft-06/schema#",
@@ -508,9 +552,6 @@
                   {
                     "$ref": "#/$defs/post_processors/gsub"
                   },
-                  {
-                    "$ref": "#/$defs/post_processors/html_to_markdown"
-                  },
                   {
                     "$ref": "#/$defs/post_processors/markdown_to_html"
                   },
@@ -600,9 +641,6 @@
                   {
                     "$ref": "#/$defs/post_processors/gsub"
                   },
-                  {
-                    "$ref": "#/$defs/post_processors/html_to_markdown"
-                  },
                   {
                     "$ref": "#/$defs/post_processors/markdown_to_html"
                   },
@@ -837,26 +875,6 @@
         ],
         "additionalProperties": true
       },
-      "html_to_markdown": {
-        "type": "object",
-        "title": "html_to_markdown",
-        "description": "Sanitize HTML then convert it to Markdown (via ReverseMarkdown).",
-        "examples": [
-          {
-            "name": "html_to_markdown"
-          }
-        ],
-        "properties": {
-          "name": {
-            "type": "string",
-            "const": "html_to_markdown"
-          }
-        },
-        "required": [
-          "name"
-        ],
-        "additionalProperties": true
-      },
       "markdown_to_html": {
         "type": "object",
         "title": "markdown_to_html",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.27.2
+  version: 0.29.1
 platform: ruby
 authors:
 - Gil Desmarais
@@ -203,20 +203,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: reverse_markdown
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: rss
   requirement: !ruby/object:Gem::Requirement
@@ -331,6 +317,7 @@ files:
 - lib/html2rss/auto_source/scraper/meta_oembed.rb
 - lib/html2rss/auto_source/scraper/microdata.rb
 - lib/html2rss/auto_source/scraper/microformats2.rb
+- lib/html2rss/auto_source/scraper/native_feed.rb
 - lib/html2rss/auto_source/scraper/schema.rb
 - lib/html2rss/auto_source/scraper/schema/category_extractor.rb
 - lib/html2rss/auto_source/scraper/schema/item_list.rb
@@ -350,10 +337,14 @@ files:
 - lib/html2rss/auto_source/segmenter/list.rb
 - lib/html2rss/auto_source/segmenter/primary_link.rb
 - lib/html2rss/auto_source/segmenter/semantic.rb
+- lib/html2rss/batch.rb
 - lib/html2rss/capture.rb
 - lib/html2rss/capture/README.md
 - lib/html2rss/channel.rb
 - lib/html2rss/cli.rb
+- lib/html2rss/cli/probe_view.rb
+- lib/html2rss/cli/render.rb
+- lib/html2rss/cli/validate.rb
 - lib/html2rss/config.rb
 - lib/html2rss/config/auto_source_contract.rb
 - lib/html2rss/config/dynamic_params.rb
@@ -364,6 +355,7 @@ files:
 - lib/html2rss/config/selectors_validator.rb
 - lib/html2rss/config/validator.rb
 - lib/html2rss/defaults.rb
+- lib/html2rss/doctor/botasaurus.rb
 - lib/html2rss/error.rb
 - lib/html2rss/feed_builder.rb
 - lib/html2rss/feed_builder/item_presentation.rb
@@ -376,6 +368,14 @@ files:
 - lib/html2rss/feed_pipeline/auto_fallback.rb
 - lib/html2rss/feed_pipeline/runtime_policy.rb
 - lib/html2rss/feed_pipeline/strategy_plan.rb
+- lib/html2rss/feed_resolution.rb
+- lib/html2rss/feed_resolution/README.md
+- lib/html2rss/feed_resolution/candidate_generator.rb
+- lib/html2rss/feed_resolution/diag.rb
+- lib/html2rss/feed_resolution/options.rb
+- lib/html2rss/feed_resolution/policy.rb
+- lib/html2rss/feed_resolution/probe.rb
+- lib/html2rss/feed_resolution/scorer.rb
 - lib/html2rss/feed_result.rb
 - lib/html2rss/hash_util.rb
 - lib/html2rss/html.rb
@@ -396,6 +396,7 @@ files:
 - lib/html2rss/html/feed_link.rb
 - lib/html2rss/html/navigator.rb
 - lib/html2rss/html/navigator/text_extractor.rb
+- lib/html2rss/html/probe.rb
 - lib/html2rss/html/rendering.rb
 - lib/html2rss/html/rendering/audio_renderer.rb
 - lib/html2rss/html/rendering/description_builder.rb
@@ -411,11 +412,18 @@ files:
 - lib/html2rss/link_destination/path_classifier.rb
 - lib/html2rss/link_destination/text_classifier.rb
 - lib/html2rss/mcp.rb
+- lib/html2rss/mcp/README.md
 - lib/html2rss/mcp/config_argument.rb
 - lib/html2rss/mcp/contract.rb
-- lib/html2rss/mcp/inspect.rb
 - lib/html2rss/mcp/outcome.rb
+- lib/html2rss/mcp/outcome/playbook.rb
+- lib/html2rss/mcp/runtime.rb
 - lib/html2rss/mcp/server.rb
+- lib/html2rss/mcp/server/tools.rb
+- lib/html2rss/page_recon.rb
+- lib/html2rss/page_recon/README.md
+- lib/html2rss/page_recon/diagnostics.rb
+- lib/html2rss/recon.rb
 - lib/html2rss/request_service.rb
 - lib/html2rss/request_service/blocked_surface.rb
 - lib/html2rss/request_service/botasaurus_contract.rb
@@ -446,6 +454,7 @@ files:
 - lib/html2rss/scoring/observation.rb
 - lib/html2rss/scoring/ranked_segment.rb
 - lib/html2rss/scoring/score.rb
+- lib/html2rss/scrape_target.rb
 - lib/html2rss/selectors.rb
 - lib/html2rss/selectors/extractors.rb
 - lib/html2rss/selectors/extractors/attribute.rb
@@ -458,7 +467,6 @@ files:
 - lib/html2rss/selectors/post_processors.rb
 - lib/html2rss/selectors/post_processors/base.rb
 - lib/html2rss/selectors/post_processors/gsub.rb
-- lib/html2rss/selectors/post_processors/html_to_markdown.rb
 - lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb
 - lib/html2rss/selectors/post_processors/html_transformers/wrap_img_in_a.rb
 - lib/html2rss/selectors/post_processors/markdown_to_html.rb
@@ -477,6 +485,14 @@ files:
 - lib/html2rss/sst/tags.rb
 - lib/html2rss/sst/text.rb
 - lib/html2rss/status.rb
+- lib/html2rss/surface_category.rb
+- lib/html2rss/syndication.rb
+- lib/html2rss/syndication/README.md
+- lib/html2rss/syndication/candidate_catalog.rb
+- lib/html2rss/syndication/discovery.rb
+- lib/html2rss/syndication/parser.rb
+- lib/html2rss/test.rb
+- lib/html2rss/test/enhance_audit.rb
 - lib/html2rss/url.rb
 - lib/html2rss/version.rb
 - lib/tasks/config_schema.rake
@@ -486,7 +502,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.27.2
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.29.1
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths: