html2rss 0.32.4 → 0.34.0

data/schema/html2rss-config.schema.json

@@ -13,7 +13,8 @@
       "properties": {
         "url": {
           "type": "string",
-          "minLength": 1
+          "minLength": 1,
+          "pattern": "^(([^:\\/?#]+):)?(\\/\\/([^\\/?#]*))?([^?#]*)(\\?([^#]*))?(#(.*))?$"
         },
         "title": {
           "type": [
@@ -31,7 +32,8 @@
           "type": [
             "null",
             "string"
-          ]
+          ],
+          "pattern": "\\A[a-z]{2}(-[A-Z]{2})?\\z"
         },
         "ttl": {
           "type": [
@@ -56,7 +58,8 @@
           "type": [
             "null",
             "string"
-          ]
+          ],
+          "pattern": "^(([^:\\/?#]+):)?(\\/\\/([^\\/?#]*))?([^?#]*)(\\?([^#]*))?(#(.*))?$"
         }
       },
       "required": [
@@ -79,8 +82,8 @@
         },
         "topics": {
           "type": "array",
+          "minItems": 1,
           "items": {
-            "minLength": 1,
             "type": "string",
             "enum": [
               "sports",
@@ -104,8 +107,7 @@
               "rights",
               "transport"
             ]
-          },
-          "minItems": 1
+          }
         }
       },
       "required": [
@@ -194,7 +196,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "wordpress_api": {
               "type": "object",
@@ -206,7 +209,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "sitemap": {
               "type": "object",
@@ -218,7 +222,14 @@
                   }
                 },
                 "min_priority": {
-                  "type": "number",
+                  "anyOf": [
+                    {
+                      "type": "integer"
+                    },
+                    {
+                      "type": "number"
+                    }
+                  ],
                   "not": {
                     "type": "null"
                   }
@@ -231,7 +242,8 @@
                   "exclusiveMinimum": 0
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "schema": {
               "type": "object",
@@ -243,7 +255,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "microdata": {
               "type": "object",
@@ -255,7 +268,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "microformats2": {
               "type": "object",
@@ -267,7 +281,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "json_state": {
               "type": "object",
@@ -279,7 +294,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "json_document": {
               "type": "object",
@@ -291,7 +307,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "xhr_articles": {
               "type": "object",
@@ -303,7 +320,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "meta_oembed": {
               "type": "object",
@@ -315,7 +333,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "semantic_html": {
               "type": "object",
@@ -325,15 +344,10 @@
                   "not": {
                     "type": "null"
                   }
-                },
-                "fallback_anchorless": {
-                  "type": "boolean",
-                  "not": {
-                    "type": "null"
-                  }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "html": {
               "type": "object",
@@ -357,18 +371,14 @@
                     "type": "null"
                   },
                   "exclusiveMinimum": 0
-                },
-                "fallback_anchorless": {
-                  "type": "boolean",
-                  "not": {
-                    "type": "null"
-                  }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             }
           },
-          "required": []
+          "required": [],
+          "additionalProperties": false
         },
         "cleanup": {
           "type": "object",
@@ -378,6 +388,12 @@
               "not": {
                 "type": "null"
               }
+            },
+            "keep_sponsored": {
+              "type": "boolean",
+              "not": {
+                "type": "null"
+              }
             }
           },
           "required": []
@@ -424,18 +440,17 @@
             "enabled": true
           },
           "semantic_html": {
-            "enabled": true,
-            "fallback_anchorless": true
+            "enabled": true
           },
           "html": {
             "enabled": true,
             "minimum_selector_frequency": 2,
-            "use_top_selectors": 5,
-            "fallback_anchorless": true
+            "use_top_selectors": 5
           }
         },
         "cleanup": {
-          "keep_different_domain": false
+          "keep_different_domain": false,
+          "keep_sponsored": false
         }
       }
     },
@@ -465,7 +480,7 @@
               "not": {
                 "type": "null"
               },
-              "description": "List-card enrichment: run Html::ArticleExtractor on each matched item node to fill missing fields from the card HTML."
+              "description": "Fills missing title, link, description, date, image and categories from each item's own markup via Html::ArticleExtractor. Explicit selectors win; no extra requests are made."
             },
             "pagination": {
               "description": "Pagination configuration or maximum page count integer.",
@@ -589,7 +604,8 @@
             },
             "content_type": {
               "type": "string",
-              "minLength": 1
+              "minLength": 1,
+              "pattern": "^[\\w-]+\\/[\\w-]+$"
             }
           },
           "required": [],
@@ -726,8 +742,7 @@
                 "auto",
                 "get",
                 "google_get",
-                "google_get_bypass",
-                "organic_get"
+                "google_get_bypass"
               ]
             },
             "max_retries": {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.32.4
+  version: 0.34.0
 platform: ruby
 authors:
 - Gil Desmarais
@@ -270,9 +270,16 @@ files:
 - lib/html2rss/article.rb
 - lib/html2rss/article/deduplicator.rb
 - lib/html2rss/article/enclosure.rb
+- lib/html2rss/article/title.rb
 - lib/html2rss/auto_source.rb
 - lib/html2rss/auto_source/README.md
 - lib/html2rss/auto_source/cleanup.rb
+- lib/html2rss/auto_source/corroboration.rb
+- lib/html2rss/auto_source/page_links.rb
+- lib/html2rss/auto_source/page_links/anchor_claims.rb
+- lib/html2rss/auto_source/page_links/card_units.rb
+- lib/html2rss/auto_source/page_links/label_binding.rb
+- lib/html2rss/auto_source/page_links/sponsored_cards.rb
 - lib/html2rss/auto_source/scraper.rb
 - lib/html2rss/auto_source/scraper/html.rb
 - lib/html2rss/auto_source/scraper/json_document.rb
@@ -324,10 +331,12 @@ files:
 - lib/html2rss/config/validation_issue.rb
 - lib/html2rss/config/validation_report.rb
 - lib/html2rss/config/validator.rb
+- lib/html2rss/contract_error.rb
 - lib/html2rss/defaults.rb
 - lib/html2rss/doctor.rb
 - lib/html2rss/doctor/botasaurus.rb
 - lib/html2rss/error.rb
+- lib/html2rss/error_kind.rb
 - lib/html2rss/feed_builder.rb
 - lib/html2rss/feed_builder/item_presentation.rb
 - lib/html2rss/feed_builder/json_feed.rb
@@ -373,14 +382,20 @@ files:
 - lib/html2rss/html/rendering/description_builder.rb
 - lib/html2rss/html/rendering/escaped_attributes.rb
 - lib/html2rss/html/rendering/image_renderer.rb
-- lib/html2rss/html/rendering/media_renderer.rb
-- lib/html2rss/html/rendering/pdf_renderer.rb
 - lib/html2rss/html/rendering/video_renderer.rb
 - lib/html2rss/html/sst_article_extractor.rb
+- lib/html2rss/html/title_pieces.rb
+- lib/html2rss/html/title_pieces/kicker.rb
+- lib/html2rss/html/title_pieces/nodes.rb
+- lib/html2rss/html/title_pieces/nokogiri_nodes.rb
+- lib/html2rss/html/title_pieces/sst_nodes.rb
+- lib/html2rss/html/visibility.rb
 - lib/html2rss/link_destination.rb
 - lib/html2rss/link_destination/destination_facts.rb
 - lib/html2rss/link_destination/noise_policy.rb
 - lib/html2rss/link_destination/path_classifier.rb
+- lib/html2rss/link_destination/sponsorship.rb
+- lib/html2rss/link_destination/sponsorship/label_text.rb
 - lib/html2rss/link_destination/text_classifier.rb
 - lib/html2rss/mcp.rb
 - lib/html2rss/mcp/README.md
@@ -391,6 +406,7 @@ files:
 - lib/html2rss/mcp/runtime.rb
 - lib/html2rss/mcp/server.rb
 - lib/html2rss/mcp/server/tools.rb
+- lib/html2rss/option.rb
 - lib/html2rss/page_recon.rb
 - lib/html2rss/page_recon/README.md
 - lib/html2rss/page_recon/diagnostics.rb
@@ -402,11 +418,17 @@ files:
 - lib/html2rss/request_service/budget.rb
 - lib/html2rss/request_service/compressed_body.rb
 - lib/html2rss/request_service/context.rb
+- lib/html2rss/request_service/cookie_jar.rb
+- lib/html2rss/request_service/cookie_store.rb
+- lib/html2rss/request_service/feed_advertisement.rb
 - lib/html2rss/request_service/httpx_strategy.rb
 - lib/html2rss/request_service/local_file_strategy.rb
 - lib/html2rss/request_service/network_guard.rb
 - lib/html2rss/request_service/policy.rb
+- lib/html2rss/request_service/replay_breaker.rb
 - lib/html2rss/request_service/response.rb
+- lib/html2rss/request_service/response_cache.rb
+- lib/html2rss/request_service/response_caching.rb
 - lib/html2rss/request_service/response_guard.rb
 - lib/html2rss/request_service/strategy.rb
 - lib/html2rss/request_session.rb
@@ -441,6 +463,7 @@ files:
 - lib/html2rss/selectors/post_processors.rb
 - lib/html2rss/selectors/post_processors/base.rb
 - lib/html2rss/selectors/post_processors/gsub.rb
+- lib/html2rss/selectors/post_processors/html_transformers/keep_playable_media.rb
 - lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb
 - lib/html2rss/selectors/post_processors/html_transformers/wrap_img_in_a.rb
 - lib/html2rss/selectors/post_processors/markdown_to_html.rb
@@ -463,14 +486,20 @@ files:
 - lib/html2rss/surface_category.rb
 - lib/html2rss/syndication.rb
 - lib/html2rss/syndication/README.md
+- lib/html2rss/syndication/alternate_links.rb
 - lib/html2rss/syndication/candidate_catalog.rb
+- lib/html2rss/syndication/channel_facts.rb
 - lib/html2rss/syndication/discovery.rb
+- lib/html2rss/syndication/feed.rb
+- lib/html2rss/syndication/item_media.rb
 - lib/html2rss/syndication/parser.rb
 - lib/html2rss/test.rb
 - lib/html2rss/test/enhance_audit.rb
+- lib/html2rss/test/item_sample.rb
 - lib/html2rss/test/policy.rb
 - lib/html2rss/url.rb
 - lib/html2rss/version.rb
+- lib/html2rss/word_count.rb
 - lib/tasks/config_schema.rake
 - schema/html2rss-config.schema.json
 homepage: https://github.com/html2rss/html2rss
@@ -478,7 +507,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.32.4
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.34.0
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:

data/lib/html2rss/html/rendering/media_renderer.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module Html
-    module Rendering
-      # Factory: picks the appropriate renderer for a given enclosure or fallback image.
-      class MediaRenderer
-        # @param enclosure [Html2rss::Article::Enclosure, nil]
-        # @param image [String, Html2rss::Url, nil] Fallback image URL
-        # @param title [String]
-        # @return [ImageRenderer, VideoRenderer, AudioRenderer, PdfRenderer, nil]
-        def self.for(enclosure:, image:, title:)
-          return ImageRenderer.new(url: image, title:) if enclosure.nil? && image
-          return nil unless enclosure
-
-          create_renderer_for_type(enclosure.type, url: enclosure.url, title:)
-        end
-
-        # @private
-        # @param type [String, nil] enclosure MIME type
-        # @param url [String, Html2rss::Url] enclosure URL
-        # @param title [String, nil] title used by image renderer
-        # @return [ImageRenderer, VideoRenderer, AudioRenderer, PdfRenderer, nil]
-        def self.create_renderer_for_type(type, url:, title:)
-          case type
-          when %r{^image/}
-            ImageRenderer.new(url:, title:)
-          when %r{^video/}
-            VideoRenderer.new(url:, type:)
-          when %r{^audio/}
-            AudioRenderer.new(url:, type:)
-          when 'application/pdf'
-            PdfRenderer.new(url:)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/html/rendering/pdf_renderer.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module Html
-    module Rendering
-      # Renders an HTML <iframe> for PDF documents.
-      class PdfRenderer
-        include EscapedAttributes
-
-        # @param url [String, Html2rss::Url] PDF URL rendered in the iframe
-        def initialize(url:)
-          @url = url
-        end
-
-        # @return [String] HTML iframe snippet for PDF rendering
-        def to_html
-          attributes = [
-            %(src="#{escaped_url}"),
-            'width="100%"',
-            'height="75vh"',
-            'sandbox=""',
-            'referrerpolicy="no-referrer"',
-            'loading="lazy"'
-          ]
-          "<iframe #{attributes.join(' ')}></iframe>"
-        end
-      end
-    end
-  end
-end