html2rss 0.22.2 → 0.24.0

data/schema/html2rss-config.schema.json

@@ -109,6 +109,31 @@
               },
               "required": []
             },
+            "sitemap": {
+              "type": "object",
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
+                },
+                "min_priority": {
+                  "type": "number",
+                  "not": {
+                    "type": "null"
+                  }
+                },
+                "max_age_days": {
+                  "type": "integer",
+                  "not": {
+                    "type": "null"
+                  },
+                  "exclusiveMinimum": 0
+                }
+              },
+              "required": []
+            },
             "schema": {
               "type": "object",
               "properties": {
@@ -133,6 +158,18 @@
               },
               "required": []
             },
+            "microformats2": {
+              "type": "object",
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
+                }
+              },
+              "required": []
+            },
             "json_state": {
               "type": "object",
               "properties": {
@@ -145,6 +182,18 @@
               },
               "required": []
             },
+            "meta_oembed": {
+              "type": "object",
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
+                }
+              },
+              "required": []
+            },
             "semantic_html": {
               "type": "object",
               "properties": {
@@ -224,15 +273,26 @@
           "wordpress_api": {
             "enabled": true
           },
+          "sitemap": {
+            "enabled": true,
+            "min_priority": 0.3,
+            "max_age_days": 30
+          },
           "schema": {
             "enabled": true
           },
           "microdata": {
             "enabled": true
           },
+          "microformats2": {
+            "enabled": true
+          },
           "json_state": {
             "enabled": true
           },
+          "meta_oembed": {
+            "enabled": true
+          },
           "semantic_html": {
             "enabled": true,
             "fallback_anchorless": true
@@ -277,18 +337,58 @@
               }
             },
             "pagination": {
-              "type": "object",
-              "properties": {
-                "max_pages": {
+              "description": "Pagination configuration or maximum page count integer.",
+              "oneOf": [
+                {
                   "type": "integer",
-                  "not": {
-                    "type": "null"
-                  },
                   "exclusiveMinimum": 0
+                },
+                {
+                  "type": "object",
+                  "properties": {
+                    "strategy": {
+                      "type": "string",
+                      "enum": [
+                        "rel_next",
+                        "custom_selector",
+                        "url_template",
+                        "offset",
+                        "json_cursor"
+                      ]
+                    },
+                    "max_pages": {
+                      "type": "integer",
+                      "exclusiveMinimum": 0
+                    },
+                    "selector": {
+                      "type": "string"
+                    },
+                    "param": {
+                      "type": "string"
+                    },
+                    "start_page": {
+                      "type": "integer"
+                    },
+                    "step": {
+                      "type": "integer",
+                      "exclusiveMinimum": 0
+                    },
+                    "start_offset": {
+                      "type": "integer"
+                    },
+                    "increment": {
+                      "type": "integer",
+                      "exclusiveMinimum": 0
+                    },
+                    "cursor_path": {
+                      "type": "string"
+                    },
+                    "next_url_path": {
+                      "type": "string"
+                    }
+                  },
+                  "additionalProperties": true
                 }
-              },
-              "required": [
-                "max_pages"
               ]
             }
           },

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.22.2
+  version: 0.24.0
 platform: ruby
 authors:
 - Gil Desmarais
@@ -273,63 +273,88 @@ files:
 - exe/html2rss
 - html2rss.gemspec
 - lib/html2rss.rb
-- lib/html2rss/articles/deduplicator.rb
+- lib/html2rss/article.rb
+- lib/html2rss/article/deduplicator.rb
+- lib/html2rss/article/enclosure.rb
 - lib/html2rss/auto_source.rb
 - lib/html2rss/auto_source/cleanup.rb
+- lib/html2rss/auto_source/discovery.rb
+- lib/html2rss/auto_source/discovery/dom_clustering.rb
+- lib/html2rss/auto_source/discovery/dom_clustering/group_scorer.rb
+- lib/html2rss/auto_source/discovery/dom_clustering/overlap_resolver.rb
+- lib/html2rss/auto_source/discovery/list_candidates.rb
+- lib/html2rss/auto_source/discovery/semantic_anchor_candidates.rb
+- lib/html2rss/auto_source/discovery/semantic_containers.rb
+- lib/html2rss/auto_source/discovery/sitemap.rb
+- lib/html2rss/auto_source/link_heuristics.rb
+- lib/html2rss/auto_source/link_heuristics/anchor_signals.rb
+- lib/html2rss/auto_source/link_heuristics/container_assessor.rb
+- lib/html2rss/auto_source/link_heuristics/container_signals.rb
+- lib/html2rss/auto_source/link_heuristics/destination_facts.rb
+- lib/html2rss/auto_source/link_heuristics/href_extractor.rb
+- lib/html2rss/auto_source/link_heuristics/path_classifier.rb
+- lib/html2rss/auto_source/link_heuristics/text_classifier.rb
 - lib/html2rss/auto_source/scraper.rb
 - lib/html2rss/auto_source/scraper/html.rb
-- lib/html2rss/auto_source/scraper/html/class_clustering.rb
 - lib/html2rss/auto_source/scraper/json_state.rb
-- lib/html2rss/auto_source/scraper/link_heuristics.rb
+- lib/html2rss/auto_source/scraper/meta_oembed.rb
 - lib/html2rss/auto_source/scraper/microdata.rb
+- lib/html2rss/auto_source/scraper/microformats2.rb
 - lib/html2rss/auto_source/scraper/schema.rb
 - lib/html2rss/auto_source/scraper/schema/category_extractor.rb
 - lib/html2rss/auto_source/scraper/schema/item_list.rb
-- lib/html2rss/auto_source/scraper/schema/list_item.rb
 - lib/html2rss/auto_source/scraper/schema/thing.rb
 - lib/html2rss/auto_source/scraper/semantic_html.rb
-- lib/html2rss/auto_source/scraper/semantic_html/anchor_selector.rb
-- lib/html2rss/auto_source/scraper/semantic_html/deduplicator.rb
+- lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb
+- lib/html2rss/auto_source/scraper/sitemap.rb
 - lib/html2rss/auto_source/scraper/wordpress_api.rb
 - lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb
+- lib/html2rss/auto_source/scraper/wordpress_api/page_scope/date_archive_range.rb
 - lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb
-- lib/html2rss/blocked_surface.rb
-- lib/html2rss/category_extractor.rb
+- lib/html2rss/channel.rb
 - lib/html2rss/cli.rb
 - lib/html2rss/config.rb
-- lib/html2rss/config/class_methods.rb
+- lib/html2rss/config/auto_source_contract.rb
 - lib/html2rss/config/dynamic_params.rb
 - lib/html2rss/config/multiple_feeds_config.rb
+- lib/html2rss/config/request_controls.rb
 - lib/html2rss/config/request_headers.rb
 - lib/html2rss/config/schema.rb
+- lib/html2rss/config/selectors_validator.rb
 - lib/html2rss/config/validator.rb
-- lib/html2rss/configuration.rb
+- lib/html2rss/defaults.rb
 - lib/html2rss/error.rb
+- lib/html2rss/feed_builder.rb
+- lib/html2rss/feed_builder/item_presentation.rb
+- lib/html2rss/feed_builder/json_feed.rb
+- lib/html2rss/feed_builder/json_feed/item.rb
+- lib/html2rss/feed_builder/rss.rb
+- lib/html2rss/feed_builder/rss/stylesheet.rb
 - lib/html2rss/feed_pipeline.rb
 - lib/html2rss/feed_pipeline/auto_fallback.rb
+- lib/html2rss/feed_pipeline/runtime_policy.rb
+- lib/html2rss/feed_pipeline/strategy_plan.rb
+- lib/html2rss/feed_result.rb
 - lib/html2rss/hash_util.rb
-- lib/html2rss/html_extractor.rb
-- lib/html2rss/html_extractor/date_extractor.rb
-- lib/html2rss/html_extractor/enclosure_extractor.rb
-- lib/html2rss/html_extractor/heading_extractor.rb
-- lib/html2rss/html_extractor/id_generator.rb
-- lib/html2rss/html_extractor/image_extractor.rb
-- lib/html2rss/html_extractor/list_candidates.rb
-- lib/html2rss/html_extractor/semantic_anchor_candidates.rb
-- lib/html2rss/html_extractor/semantic_containers.rb
-- lib/html2rss/html_extractor/text_extractor.rb
-- lib/html2rss/html_navigator.rb
-- lib/html2rss/json_feed_builder.rb
-- lib/html2rss/json_feed_builder/item.rb
-- lib/html2rss/rendering.rb
-- lib/html2rss/rendering/audio_renderer.rb
-- lib/html2rss/rendering/description_builder.rb
-- lib/html2rss/rendering/image_renderer.rb
-- lib/html2rss/rendering/media_renderer.rb
-- lib/html2rss/rendering/pdf_renderer.rb
-- lib/html2rss/rendering/video_renderer.rb
-- lib/html2rss/request_controls.rb
+- lib/html2rss/html.rb
+- lib/html2rss/html/article_extractor.rb
+- lib/html2rss/html/article_extractor/category_extractor.rb
+- lib/html2rss/html/article_extractor/date_extractor.rb
+- lib/html2rss/html/article_extractor/enclosure_extractor.rb
+- lib/html2rss/html/article_extractor/heading_extractor.rb
+- lib/html2rss/html/article_extractor/id_generator.rb
+- lib/html2rss/html/article_extractor/image_extractor.rb
+- lib/html2rss/html/navigator.rb
+- lib/html2rss/html/navigator/text_extractor.rb
+- lib/html2rss/html/rendering.rb
+- lib/html2rss/html/rendering/audio_renderer.rb
+- lib/html2rss/html/rendering/description_builder.rb
+- lib/html2rss/html/rendering/image_renderer.rb
+- lib/html2rss/html/rendering/media_renderer.rb
+- lib/html2rss/html/rendering/pdf_renderer.rb
+- lib/html2rss/html/rendering/video_renderer.rb
 - lib/html2rss/request_service.rb
+- lib/html2rss/request_service/blocked_surface.rb
 - lib/html2rss/request_service/botasaurus_contract.rb
 - lib/html2rss/request_service/botasaurus_strategy.rb
 - lib/html2rss/request_service/browserless_strategy.rb
@@ -337,28 +362,30 @@ files:
 - lib/html2rss/request_service/context.rb
 - lib/html2rss/request_service/faraday_strategy.rb
 - lib/html2rss/request_service/local_file_strategy.rb
+- lib/html2rss/request_service/network_guard.rb
 - lib/html2rss/request_service/policy.rb
 - lib/html2rss/request_service/puppet_commander.rb
+- lib/html2rss/request_service/puppet_commander/navigation_guards.rb
+- lib/html2rss/request_service/puppet_commander/preload_runner.rb
 - lib/html2rss/request_service/response.rb
 - lib/html2rss/request_service/response_guard.rb
 - lib/html2rss/request_service/strategy.rb
 - lib/html2rss/request_session.rb
-- lib/html2rss/request_session/rel_next_pager.rb
-- lib/html2rss/request_session/runtime_input.rb
-- lib/html2rss/request_session/runtime_policy.rb
-- lib/html2rss/rss_builder.rb
-- lib/html2rss/rss_builder/article.rb
-- lib/html2rss/rss_builder/channel.rb
-- lib/html2rss/rss_builder/enclosure.rb
-- lib/html2rss/rss_builder/stylesheet.rb
+- lib/html2rss/request_session/pager.rb
+- lib/html2rss/request_session/pager/base.rb
+- lib/html2rss/request_session/pager/custom_selector.rb
+- lib/html2rss/request_session/pager/json_cursor.rb
+- lib/html2rss/request_session/pager/offset.rb
+- lib/html2rss/request_session/pager/rel_next.rb
+- lib/html2rss/request_session/pager/url_template.rb
 - lib/html2rss/selectors.rb
-- lib/html2rss/selectors/config.rb
 - lib/html2rss/selectors/extractors.rb
 - lib/html2rss/selectors/extractors/attribute.rb
 - lib/html2rss/selectors/extractors/href.rb
 - lib/html2rss/selectors/extractors/html.rb
 - lib/html2rss/selectors/extractors/static.rb
 - lib/html2rss/selectors/extractors/text.rb
+- lib/html2rss/selectors/item_scope.rb
 - lib/html2rss/selectors/object_to_xml_converter.rb
 - lib/html2rss/selectors/post_processors.rb
 - lib/html2rss/selectors/post_processors/base.rb
@@ -372,6 +399,7 @@ files:
 - lib/html2rss/selectors/post_processors/sanitize_html.rb
 - lib/html2rss/selectors/post_processors/substring.rb
 - lib/html2rss/selectors/post_processors/template.rb
+- lib/html2rss/status.rb
 - lib/html2rss/url.rb
 - lib/html2rss/version.rb
 - lib/tasks/config_schema.rake
@@ -381,7 +409,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.22.2
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.24.0
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
@@ -397,7 +425,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.16
 specification_version: 4
 summary: Generates RSS feeds from websites by scraping a URL and using CSS selectors
   to extract item.

data/lib/html2rss/auto_source/scraper/html/class_clustering.rb

@@ -1,196 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    module Scraper
-      class Html
-        ##
-        # ClassClustering clusters DOM elements on anchorless pages by class lists and scores
-        # candidate groups to find the best list of content cards/articles.
-        # rubocop:disable Metrics/ClassLength
-        class ClassClustering
-          # Node tags considered layout containers
-          LAYOUT_TAG_NAMES = Set['div', 'section', 'article', 'li', 'ul', 'ol'].freeze
-          # HTML/layout tags excluded from candidate nodes
-          EXCLUDED_TAGS = Set['html', 'body', 'nav', 'footer', 'header', 'svg', 'script', 'style'].freeze
-
-          class << self
-            ##
-            # Clusters elements in parsed_body and returns the best set of content card nodes.
-            #
-            # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
-            # @param minimum_selector_frequency [Integer] minimum frequency for class groups
-            # @return [Array<Nokogiri::XML::Node>] candidate nodes of the top-scoring class group
-            def call(parsed_body, minimum_selector_frequency:)
-              new(parsed_body, minimum_selector_frequency:).call
-            end
-          end
-
-          # @param parsed_body [Nokogiri::HTML::Document]
-          # @param minimum_selector_frequency [Integer]
-          def initialize(parsed_body, minimum_selector_frequency:)
-            @parsed_body = parsed_body
-            @minimum_frequency = minimum_selector_frequency
-            @text_words = {}.compare_by_identity
-            @has_date = {}.compare_by_identity
-          end
-
-          # @return [Array<Nokogiri::XML::Node>]
-          def call
-            candidate_groups = collect_candidate_groups
-            return [] if candidate_groups.empty?
-
-            non_containers = filter_containers(candidate_groups)
-            final_groups = filter_1_to_1_overlap(non_containers)
-
-            select_best_group(final_groups)
-          end
-
-          private
-
-          def collect_candidate_groups
-            class_groups = Hash.new { |h, k| h[k] = [] }
-            cache = {}.compare_by_identity
-
-            @parsed_body.css('[class]').each { |node| add_node_to_groups(node, class_groups, cache) }
-
-            class_groups.select { |_, nodes| nodes.size >= @minimum_frequency }
-          end
-
-          def add_node_to_groups(node, class_groups, cache)
-            return if EXCLUDED_TAGS.include?(node.name) || HtmlExtractor.ignored_container_path?(node, cache)
-
-            cls = normalize_class(node['class'])
-            class_groups[cls] << node unless cls.empty?
-          end
-
-          def normalize_class(class_attr)
-            class_str = class_attr.to_s.strip
-            return '' if class_str.empty?
-
-            # Bypass split/sort/join allocation for single-class lists
-            if class_str.include?(' ')
-              class_str.split(/\s+/).sort.join(' ')
-            else
-              class_str
-            end
-          end
-
-          # Discard group A if any node of A contains > 1 node of another group B
-          def filter_containers(groups)
-            groups.reject do |cls_a, nodes_a|
-              groups.any? { |cls_b, nodes_b| cls_a != cls_b && container_of?(nodes_a, nodes_b) }
-            end
-          end
-
-          # rubocop:disable Metrics/MethodLength
-          def container_of?(nodes_a, nodes_b)
-            return false unless LAYOUT_TAG_NAMES.include?(nodes_b.first.name)
-
-            nodes_a.any? do |node_a|
-              count = 0
-              nodes_b.each do |node_b|
-                next if node_a == node_b
-
-                if HtmlNavigator.descendant_of?(node_b, node_a)
-                  count += 1
-                  break if count > 1
-                end
-              end
-              count > 1
-            end
-          end
-          # rubocop:enable Metrics/MethodLength
-
-          # If group A contains group B, and they have the same size:
-          # - If B (the descendant) contains >= 80% of A's words, AND B's tag is div/section/article,
-          #   B is the actual content card. Discard A.
-          # - Otherwise, B is a sub-element (header, metadata line, button). Discard B.
-          def filter_1_to_1_overlap(groups)
-            discarded = Set.new
-            groups.each_key do |cls_a|
-              groups.each_key do |cls_b|
-                next if cls_a == cls_b || discarded.include?(cls_a) || discarded.include?(cls_b)
-
-                resolve_1_to_1_overlap(cls_a, cls_b, groups, discarded)
-              end
-            end
-
-            groups.except(*discarded)
-          end
-
-          def resolve_1_to_1_overlap(cls_a, cls_b, groups, discarded)
-            nodes_a = groups[cls_a]
-            nodes_b = groups[cls_b]
-            return if nodes_a.size != nodes_b.size
-            return unless nodes_a.zip(nodes_b).all? { |a, b| a != b && HtmlNavigator.descendant_of?(b, a) }
-
-            discarded << (keep_descendant?(nodes_a, nodes_b) ? cls_a : cls_b)
-          end
-
-          def keep_descendant?(nodes_a, nodes_b)
-            avg_words(nodes_b) >= 0.8 * avg_words(nodes_a) &&
-              LAYOUT_TAG_NAMES.include?(nodes_b.first.name)
-          end
-
-          def select_best_group(groups)
-            best_nodes = []
-            best_score = -1
-
-            groups.each_value do |nodes|
-              score = score_group(nodes)
-              next if score.negative?
-
-              (best_nodes = nodes) && (best_score = score) if score > best_score
-            end
-
-            best_nodes
-          end
-
-          def score_group(nodes)
-            avg_w = avg_words(nodes)
-            return -1 if avg_w < 5
-
-            score = nodes.size + (avg_w / 5.0)
-            score += 20 if nodes_heading?(nodes)
-            score += 20 if nodes_time?(nodes)
-            score += 40 if nodes_date?(nodes)
-            score
-          end
-
-          def nodes_heading?(nodes)
-            nodes.any? do |n|
-              n.at_css(HtmlExtractor::HEADING_TAGS.join(',')) ||
-                n.at_css('.font-bold, .font-semibold')
-            end
-          end
-
-          def nodes_time?(nodes)
-            nodes.any? { |n| n.at_css('time, [datetime]') }
-          end
-
-          def nodes_date?(nodes)
-            nodes.any? { |n| date?(n) }
-          end
-
-          def avg_words(nodes)
-            nodes.sum { |n| text_words(n) } / nodes.size.to_f
-          end
-
-          def text_words(node)
-            @text_words[node] ||= HtmlExtractor.extract_visible_text(node).to_s.scan(/\p{Alnum}+/).size
-          end
-
-          def date?(node)
-            @has_date[node] ||= begin
-              text = HtmlExtractor.extract_visible_text(node).to_s
-              text.match?(%r{\b\d{4}[-/]\d{2}[-/]\d{2}\b}) ||
-                text.match?(/\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/i)
-            end
-          end
-          # rubocop:enable Metrics/ClassLength
-        end
-      end
-    end
-  end
-end