html2rss 0.26.0 → 0.27.1

data/lib/html2rss/request_service.rb

@@ -33,8 +33,13 @@ module Html2rss
     class BlockedSurfaceDetected < Html2rss::Error; end
     # Raised when a request times out.
     class RequestTimedOut < Html2rss::Error; end
+
     # Raised when Botasaurus configuration is missing or invalid.
-    class BotasaurusConfigurationError < Html2rss::Error; end
+    class BotasaurusConfigurationError < Html2rss::Error
+      # Short empty-feed guidance owned once (composed by {Html2rss::NoFeedItemsExtracted}).
+      EMPTY_FEED_HINT = 'Configure BOTASAURUS_SCRAPER_URL to enable the Botasaurus request strategy.'
+    end
+
     # Raised when the Botasaurus service cannot be reached (network / DNS / SSL).
     class BotasaurusConnectionFailed < Html2rss::Error; end
     # Raised when Botasaurus responds but the scrape fails (upstream error, bad payload).

data/lib/html2rss/selectors.rb

@@ -119,7 +119,8 @@ module Html2rss
         article_tag,
         base_url:,
         selected_anchor:,
-        fallback_anchorless: true
+        fallback_anchorless: true,
+        time_zone: @time_zone
       )
       return article_hash unless extracted
 

data/lib/html2rss/status.rb

@@ -8,7 +8,8 @@ module Html2rss
   # Stable telemetry payload for cross-repo consumers (e.g. html2rss-web observability).
   # Tallies and counters are validated and frozen at construction (including Marshal load).
   Status = Data.define(
-    :version, :scraper_tallies, :dedup_dropped, :selected_strategy, :attempt_count, :strategy_attempts
+    :version, :scraper_tallies, :dedup_dropped, :selected_strategy, :attempt_count,
+    :strategy_attempts, :admission_drops
   ) do
     class << self
       ##
@@ -19,8 +20,11 @@ module Html2rss
       # @param selected_strategy [Symbol, nil] concrete strategy that succeeded under +:auto+ (else +nil+)
       # @param attempt_count [Integer] auto-fallback attempt count (0 when not under +:auto+)
       # @param strategy_attempts [Array<Hash>] auto-fallback attempt hashes (empty outside +:auto+)
+      # @param admission_drops [Hash{String => Integer}] Cleanup reason → count (empty when unused)
       # @return [Html2rss::Status]
-      def build(articles:, dedup_dropped: 0, selected_strategy: nil, attempt_count: 0, strategy_attempts: [])
+      # rubocop:disable Metrics/ParameterLists -- Status kwargs stay co-located
+      def build(articles:, dedup_dropped: 0, selected_strategy: nil, attempt_count: 0,
+                strategy_attempts: [], admission_drops: {})
         tallies = articles.filter_map(&:scraper).tally.transform_keys { |klass| scraper_name(klass) }
         new(
           version: Html2rss::VERSION,
@@ -28,9 +32,11 @@ module Html2rss
           dedup_dropped:,
           selected_strategy:,
           attempt_count:,
-          strategy_attempts:
+          strategy_attempts:,
+          admission_drops:
         )
       end
+      # rubocop:enable Metrics/ParameterLists
 
       ##
       # @param klass [Class, #to_s] scraper class
@@ -47,9 +53,11 @@ module Html2rss
     # @param selected_strategy [Symbol, nil]
     # @param attempt_count [Integer]
     # @param strategy_attempts [Array<Hash>]
+    # @param admission_drops [Hash{String => Integer}]
     # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength -- Status Data.define members
     def initialize(
-      version:, scraper_tallies:, dedup_dropped:, selected_strategy: nil, attempt_count: 0, strategy_attempts: []
+      version:, scraper_tallies:, dedup_dropped:, selected_strategy: nil, attempt_count: 0,
+      strategy_attempts: [], admission_drops: {}
     )
       dedup = Integer(dedup_dropped)
       attempts = Integer(attempt_count)
@@ -61,7 +69,8 @@ module Html2rss
         dedup_dropped: dedup,
         selected_strategy:,
         attempt_count: attempts,
-        strategy_attempts: freeze_attempts(strategy_attempts)
+        strategy_attempts: freeze_attempts(strategy_attempts),
+        admission_drops: freeze_tallies(admission_drops)
       )
     end
     # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
@@ -69,11 +78,13 @@ module Html2rss
     ##
     # Observability hash for web (+scraper_status+). Omits empty/absent optional keys:
     # +:scraper_tallies+ when empty, +:selected_strategy+ when +nil+, +:attempt_count+ when zero,
-    # +:strategy_attempts+ when empty.
+    # +:strategy_attempts+ / +:admission_drops+ when empty.
     # Data members remain available via readers even when omitted here.
     #
     # @return [Hash{Symbol => Object}] always +:version+ (String), +:dedup_dropped+ (Integer);
-    #   optionally +:scraper_tallies+, +:selected_strategy+, +:attempt_count+, +:strategy_attempts+
+    #   optionally +:scraper_tallies+, +:selected_strategy+, +:attempt_count+,
+    #   +:strategy_attempts+, +:admission_drops+
+    # rubocop:disable Metrics/AbcSize -- omit-empty optional keys stay explicit
     def to_h
       {
         version:,
@@ -81,9 +92,11 @@ module Html2rss
         **(scraper_tallies.any? ? { scraper_tallies: } : {}),
         **(selected_strategy.nil? ? {} : { selected_strategy: }),
         **(attempt_count.positive? ? { attempt_count: } : {}),
-        **(strategy_attempts.any? ? { strategy_attempts: } : {})
+        **(strategy_attempts.any? ? { strategy_attempts: } : {}),
+        **(admission_drops.any? ? { admission_drops: } : {})
       }
     end
+    # rubocop:enable Metrics/AbcSize
 
     ##
     # Formats the RSS +generator+ string and JSON Feed +user_comment+.
@@ -101,11 +114,14 @@ module Html2rss
     private
 
     def marshal_dump
-      [version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts]
+      [version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts,
+       admission_drops]
     end
 
-    def marshal_load((version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts))
-      initialize(version:, scraper_tallies:, dedup_dropped:, selected_strategy:, attempt_count:, strategy_attempts:)
+    def marshal_load((version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count,
+                      strategy_attempts, admission_drops))
+      initialize(version:, scraper_tallies:, dedup_dropped:, selected_strategy:, attempt_count:,
+                 strategy_attempts:, admission_drops: admission_drops || {})
     end
 
     def freeze_tallies(tallies)

data/lib/html2rss/url.rb

@@ -2,6 +2,7 @@
 
 require 'addressable/uri'
 require 'cgi'
+require 'ipaddr'
 require 'nokogiri'
 
 module Html2rss
@@ -152,6 +153,16 @@ module Html2rss
     # @return [String, nil] URI host component
     def host = @uri.host
 
+    # Registrable domain (eTLD+1). Same-site hosts like www.wp.pl and wiadomosci.wp.pl
+    # share `wp.pl`. IP literals stay the host so 1.2.3.4 is not treated as `3.4`.
+    #
+    # @return [String, nil]
+    def domain
+      return host if ip_address_host?
+
+      @uri.domain || host
+    end
+
     # @return [Integer, nil] URI port component
     def port = @uri.port
 
@@ -281,6 +292,15 @@ module Html2rss
     #
     # @return [String] the debug representation
     def inspect = "#<#{self.class}:#{object_id} @uri=#{@uri.inspect}>"
+
+    private
+
+    def ip_address_host?
+      IPAddr.new(host.to_s)
+      true
+    rescue IPAddr::Error
+      false
+    end
   end
   # rubocop:enable Metrics/ClassLength
 end

data/lib/html2rss/version.rb

@@ -2,6 +2,6 @@
 
 module Html2rss
   # Current application version.
-  VERSION = '0.26.0'
+  VERSION = '0.27.1'
   public_constant :VERSION
 end

data/lib/html2rss.rb

@@ -70,11 +70,35 @@ module Html2rss
 
   # rubocop:disable Metrics/ParameterLists
 
+  ##
+  # Scrapes the provided URL without hand-written selectors and returns {FeedResult}.
+  #
+  # Prefer this when callers need {FeedResult#status} (MCP envelope payload, CLI +--explain+).
+  #
+  # @param url [String] source page URL
+  # @param strategy [Symbol] request strategy to use
+  # @param items_selector [String, nil] optional selector hint for item extraction
+  # @param max_redirects [Integer, nil] optional redirect limit override
+  # @param max_requests [Integer] optional request budget override (default: 4 for sitemap sub-fetches)
+  # @param local_file_path [String, nil] optional local HTML file path
+  # @param limit [Integer, nil] max articles to keep (default: {AutoSource::DEFAULT_LIMIT})
+  # @return [Html2rss::FeedResult]
+  def self.auto_feed_result(url,
+                            strategy: :auto,
+                            items_selector: nil,
+                            max_redirects: nil,
+                            max_requests: 4,
+                            local_file_path: nil,
+                            limit: nil)
+    feed_result(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
+                                         local_file_path:, limit:))
+  end
+
   ##
   # Scrapes the provided URL without hand-written selectors and returns an RSS object.
   #
   # Builds an auto_source config, then FeedPipeline runs structured scrapers and
-  # (when needed) the SST heuristic path — see docs/auto_source.md.
+  # (when needed) the SST heuristic path — see lib/html2rss/auto_source/README.md.
   #
   # @param url [String] source page URL
   # @param strategy [Symbol] request strategy to use
@@ -91,14 +115,14 @@ module Html2rss
                        max_requests: 4,
                        local_file_path: nil,
                        limit: nil)
-    feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
-                                  local_file_path:, limit:))
+    auto_feed_result(url, strategy:, items_selector:, max_redirects:, max_requests:,
+                          local_file_path:, limit:).to_rss
   end
 
   ##
   # Scrapes the provided URL without hand-written selectors and returns a JSONFeed 1.1 hash.
   #
-  # Same auto_source pipeline as {.auto_source}; see docs/auto_source.md.
+  # Same auto_source pipeline as {.auto_source}; see lib/html2rss/auto_source/README.md.
   #
   # @param url [String] source page URL
   # @param strategy [Symbol] request strategy to use
@@ -115,8 +139,8 @@ module Html2rss
                           max_requests: 4,
                           local_file_path: nil,
                           limit: nil)
-    json_feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
-                                       local_file_path:, limit:))
+    auto_feed_result(url, strategy:, items_selector:, max_redirects:, max_requests:,
+                          local_file_path:, limit:).to_json_feed
   end
 
   ##

data/schema/html2rss-config.schema.json

@@ -66,6 +66,17 @@
     "directory": {
       "type": "object",
       "properties": {
+        "title": {
+          "type": "string",
+          "minLength": 1
+        },
+        "summary": {
+          "type": [
+            "null",
+            "string"
+          ],
+          "maxLength": 160
+        },
         "topics": {
           "type": "array",
           "items": {
@@ -92,7 +103,9 @@
           "minItems": 1
         }
       },
-      "required": []
+      "required": [
+        "title"
+      ]
     },
     "headers": {
       "type": "object",
@@ -684,7 +697,8 @@
               "not": {
                 "type": "null"
               },
-              "exclusiveMinimum": 0
+              "minimum": 1,
+              "maximum": 20
             },
             "scroll": {
               "type": "boolean",
@@ -692,12 +706,6 @@
                 "type": "null"
               }
             },
-            "scroll_to_bottom": {
-              "type": "boolean",
-              "not": {
-                "type": "null"
-              }
-            },
             "block_images": {
               "type": "boolean",
               "not": {
@@ -737,13 +745,28 @@
               "minLength": 1
             },
             "window_size": {
-              "type": "array",
-              "items": {
-                "minLength": 2,
-                "maxLength": 2,
-                "type": "integer",
-                "exclusiveMinimum": 0
-              }
+              "type": "object",
+              "properties": {
+                "width": {
+                  "type": "integer",
+                  "not": {
+                    "type": "null"
+                  },
+                  "exclusiveMinimum": 0
+                },
+                "height": {
+                  "type": "integer",
+                  "not": {
+                    "type": "null"
+                  },
+                  "exclusiveMinimum": 0
+                }
+              },
+              "required": [
+                "width",
+                "height"
+              ],
+              "additionalProperties": false
             },
             "lang": {
               "type": "string",
@@ -756,7 +779,8 @@
               "type": "object"
             }
           },
-          "required": []
+          "required": [],
+          "additionalProperties": false
         },
         "local_file_path": {
           "type": "string",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.26.0
+  version: 0.27.1
 platform: ruby
 authors:
 - Gil Desmarais
@@ -119,14 +119,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: mime-types
   requirement: !ruby/object:Gem::Requirement
@@ -319,6 +319,7 @@ files:
 - lib/html2rss/article/deduplicator.rb
 - lib/html2rss/article/enclosure.rb
 - lib/html2rss/auto_source.rb
+- lib/html2rss/auto_source/README.md
 - lib/html2rss/auto_source/cleanup.rb
 - lib/html2rss/auto_source/scraper.rb
 - lib/html2rss/auto_source/scraper/html.rb
@@ -350,6 +351,7 @@ files:
 - lib/html2rss/auto_source/segmenter/primary_link.rb
 - lib/html2rss/auto_source/segmenter/semantic.rb
 - lib/html2rss/capture.rb
+- lib/html2rss/capture/README.md
 - lib/html2rss/channel.rb
 - lib/html2rss/cli.rb
 - lib/html2rss/config.rb
@@ -370,6 +372,7 @@ files:
 - lib/html2rss/feed_builder/rss.rb
 - lib/html2rss/feed_builder/rss/stylesheet.rb
 - lib/html2rss/feed_pipeline.rb
+- lib/html2rss/feed_pipeline/README.md
 - lib/html2rss/feed_pipeline/auto_fallback.rb
 - lib/html2rss/feed_pipeline/runtime_policy.rb
 - lib/html2rss/feed_pipeline/strategy_plan.rb
@@ -386,8 +389,11 @@ files:
 - lib/html2rss/html/article_rules.rb
 - lib/html2rss/html/article_rules/category.rb
 - lib/html2rss/html/article_rules/date.rb
+- lib/html2rss/html/article_rules/description.rb
 - lib/html2rss/html/article_rules/enclosure.rb
 - lib/html2rss/html/article_rules/image.rb
+- lib/html2rss/html/card_walk.rb
+- lib/html2rss/html/feed_link.rb
 - lib/html2rss/html/navigator.rb
 - lib/html2rss/html/navigator/text_extractor.rb
 - lib/html2rss/html/rendering.rb
@@ -405,12 +411,17 @@ files:
 - lib/html2rss/link_destination/path_classifier.rb
 - lib/html2rss/link_destination/text_classifier.rb
 - lib/html2rss/mcp.rb
+- lib/html2rss/mcp/config_argument.rb
+- lib/html2rss/mcp/contract.rb
+- lib/html2rss/mcp/inspect.rb
+- lib/html2rss/mcp/outcome.rb
 - lib/html2rss/mcp/server.rb
 - lib/html2rss/request_service.rb
 - lib/html2rss/request_service/blocked_surface.rb
 - lib/html2rss/request_service/botasaurus_contract.rb
 - lib/html2rss/request_service/botasaurus_strategy.rb
 - lib/html2rss/request_service/budget.rb
+- lib/html2rss/request_service/compressed_body.rb
 - lib/html2rss/request_service/context.rb
 - lib/html2rss/request_service/faraday_strategy.rb
 - lib/html2rss/request_service/local_file_strategy.rb
@@ -475,7 +486,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.26.0
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.27.1
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths: