html2rss 0.24.0 → 0.26.0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.24.0
+  version: 0.26.0
 platform: ruby
 authors:
 - Gil Desmarais
@@ -113,6 +113,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mcp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: mime-types
   requirement: !ruby/object:Gem::Requirement
@@ -148,19 +162,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: puppeteer-ruby
+  name: rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rackup
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: regexp_parser
   requirement: !ruby/object:Gem::Requirement
@@ -245,6 +273,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -278,25 +320,13 @@ files:
 - lib/html2rss/article/enclosure.rb
 - lib/html2rss/auto_source.rb
 - lib/html2rss/auto_source/cleanup.rb
-- lib/html2rss/auto_source/discovery.rb
-- lib/html2rss/auto_source/discovery/dom_clustering.rb
-- lib/html2rss/auto_source/discovery/dom_clustering/group_scorer.rb
-- lib/html2rss/auto_source/discovery/dom_clustering/overlap_resolver.rb
-- lib/html2rss/auto_source/discovery/list_candidates.rb
-- lib/html2rss/auto_source/discovery/semantic_anchor_candidates.rb
-- lib/html2rss/auto_source/discovery/semantic_containers.rb
-- lib/html2rss/auto_source/discovery/sitemap.rb
-- lib/html2rss/auto_source/link_heuristics.rb
-- lib/html2rss/auto_source/link_heuristics/anchor_signals.rb
-- lib/html2rss/auto_source/link_heuristics/container_assessor.rb
-- lib/html2rss/auto_source/link_heuristics/container_signals.rb
-- lib/html2rss/auto_source/link_heuristics/destination_facts.rb
-- lib/html2rss/auto_source/link_heuristics/href_extractor.rb
-- lib/html2rss/auto_source/link_heuristics/path_classifier.rb
-- lib/html2rss/auto_source/link_heuristics/text_classifier.rb
 - lib/html2rss/auto_source/scraper.rb
 - lib/html2rss/auto_source/scraper/html.rb
 - lib/html2rss/auto_source/scraper/json_state.rb
+- lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb
+- lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb
+- lib/html2rss/auto_source/scraper/json_state/document_scanner.rb
+- lib/html2rss/auto_source/scraper/json_state/value_finder.rb
 - lib/html2rss/auto_source/scraper/meta_oembed.rb
 - lib/html2rss/auto_source/scraper/microdata.rb
 - lib/html2rss/auto_source/scraper/microformats2.rb
@@ -307,10 +337,19 @@ files:
 - lib/html2rss/auto_source/scraper/semantic_html.rb
 - lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb
 - lib/html2rss/auto_source/scraper/sitemap.rb
+- lib/html2rss/auto_source/scraper/sitemap/parser.rb
 - lib/html2rss/auto_source/scraper/wordpress_api.rb
 - lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb
 - lib/html2rss/auto_source/scraper/wordpress_api/page_scope/date_archive_range.rb
 - lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb
+- lib/html2rss/auto_source/scraper/xhr_articles.rb
+- lib/html2rss/auto_source/segment.rb
+- lib/html2rss/auto_source/segmenter.rb
+- lib/html2rss/auto_source/segmenter/cluster.rb
+- lib/html2rss/auto_source/segmenter/list.rb
+- lib/html2rss/auto_source/segmenter/primary_link.rb
+- lib/html2rss/auto_source/segmenter/semantic.rb
+- lib/html2rss/capture.rb
 - lib/html2rss/channel.rb
 - lib/html2rss/cli.rb
 - lib/html2rss/config.rb
@@ -344,29 +383,39 @@ files:
 - lib/html2rss/html/article_extractor/heading_extractor.rb
 - lib/html2rss/html/article_extractor/id_generator.rb
 - lib/html2rss/html/article_extractor/image_extractor.rb
+- lib/html2rss/html/article_rules.rb
+- lib/html2rss/html/article_rules/category.rb
+- lib/html2rss/html/article_rules/date.rb
+- lib/html2rss/html/article_rules/enclosure.rb
+- lib/html2rss/html/article_rules/image.rb
 - lib/html2rss/html/navigator.rb
 - lib/html2rss/html/navigator/text_extractor.rb
 - lib/html2rss/html/rendering.rb
 - lib/html2rss/html/rendering/audio_renderer.rb
 - lib/html2rss/html/rendering/description_builder.rb
+- lib/html2rss/html/rendering/escaped_attributes.rb
 - lib/html2rss/html/rendering/image_renderer.rb
 - lib/html2rss/html/rendering/media_renderer.rb
 - lib/html2rss/html/rendering/pdf_renderer.rb
 - lib/html2rss/html/rendering/video_renderer.rb
+- lib/html2rss/html/sst_article_extractor.rb
+- lib/html2rss/link_destination.rb
+- lib/html2rss/link_destination/destination_facts.rb
+- lib/html2rss/link_destination/noise_policy.rb
+- lib/html2rss/link_destination/path_classifier.rb
+- lib/html2rss/link_destination/text_classifier.rb
+- lib/html2rss/mcp.rb
+- lib/html2rss/mcp/server.rb
 - lib/html2rss/request_service.rb
 - lib/html2rss/request_service/blocked_surface.rb
 - lib/html2rss/request_service/botasaurus_contract.rb
 - lib/html2rss/request_service/botasaurus_strategy.rb
-- lib/html2rss/request_service/browserless_strategy.rb
 - lib/html2rss/request_service/budget.rb
 - lib/html2rss/request_service/context.rb
 - lib/html2rss/request_service/faraday_strategy.rb
 - lib/html2rss/request_service/local_file_strategy.rb
 - lib/html2rss/request_service/network_guard.rb
 - lib/html2rss/request_service/policy.rb
-- lib/html2rss/request_service/puppet_commander.rb
-- lib/html2rss/request_service/puppet_commander/navigation_guards.rb
-- lib/html2rss/request_service/puppet_commander/preload_runner.rb
 - lib/html2rss/request_service/response.rb
 - lib/html2rss/request_service/response_guard.rb
 - lib/html2rss/request_service/strategy.rb
@@ -378,6 +427,14 @@ files:
 - lib/html2rss/request_session/pager/offset.rb
 - lib/html2rss/request_session/pager/rel_next.rb
 - lib/html2rss/request_session/pager/url_template.rb
+- lib/html2rss/scoring.rb
+- lib/html2rss/scoring/cluster_scorer.rb
+- lib/html2rss/scoring/container_assessor.rb
+- lib/html2rss/scoring/engine.rb
+- lib/html2rss/scoring/link_resolver.rb
+- lib/html2rss/scoring/observation.rb
+- lib/html2rss/scoring/ranked_segment.rb
+- lib/html2rss/scoring/score.rb
 - lib/html2rss/selectors.rb
 - lib/html2rss/selectors/extractors.rb
 - lib/html2rss/selectors/extractors/attribute.rb
@@ -399,6 +456,15 @@ files:
 - lib/html2rss/selectors/post_processors/sanitize_html.rb
 - lib/html2rss/selectors/post_processors/substring.rb
 - lib/html2rss/selectors/post_processors/template.rb
+- lib/html2rss/selectors/schema_doc.rb
+- lib/html2rss/sst.rb
+- lib/html2rss/sst/attrs.rb
+- lib/html2rss/sst/document.rb
+- lib/html2rss/sst/index.rb
+- lib/html2rss/sst/node.rb
+- lib/html2rss/sst/normalizer.rb
+- lib/html2rss/sst/tags.rb
+- lib/html2rss/sst/text.rb
 - lib/html2rss/status.rb
 - lib/html2rss/url.rb
 - lib/html2rss/version.rb
@@ -409,7 +475,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.24.0
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.26.0
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:

data/lib/html2rss/auto_source/discovery/dom_clustering/group_scorer.rb

@@ -1,80 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    module Discovery
-      class DomClustering
-        ##
-        # Scores candidate DOM groups using heading, time, date, and word-count signals.
-        class GroupScorer
-          def initialize
-            @text_words = {}.compare_by_identity
-            @has_date = {}.compare_by_identity
-          end
-
-          # @param groups [Hash{String => Array<Nokogiri::XML::Node>}] candidate DOM groups
-          # @return [Array<Nokogiri::XML::Node>] nodes from the highest-scoring group
-          def select_best_group(groups)
-            best_nodes = []
-            best_score = -1
-
-            groups.each_value do |nodes|
-              score = score_group(nodes)
-              next if score.negative?
-
-              (best_nodes = nodes) && (best_score = score) if score > best_score
-            end
-
-            best_nodes
-          end
-
-          # @param nodes [Array<Nokogiri::XML::Node>]
-          # @return [Float] average visible word count across nodes
-          def avg_words(nodes)
-            nodes.sum { |n| text_words(n) } / nodes.size.to_f
-          end
-
-          private
-
-          def score_group(nodes)
-            avg_w = avg_words(nodes)
-            return -1 if avg_w < 5
-
-            score = nodes.size + (avg_w / 5.0)
-            score += 20 if nodes_heading?(nodes)
-            score += 20 if nodes_time?(nodes)
-            score += 40 if nodes_date?(nodes)
-            score
-          end
-
-          def nodes_heading?(nodes)
-            nodes.any? do |n|
-              n.at_css(Html2rss::Html::Navigator::HEADING_TAGS.join(',')) ||
-                n.at_css('.font-bold, .font-semibold')
-            end
-          end
-
-          def nodes_time?(nodes)
-            nodes.any? { |n| n.at_css('time, [datetime]') }
-          end
-
-          def nodes_date?(nodes)
-            nodes.any? { |n| date?(n) }
-          end
-
-          def text_words(node)
-            @text_words[node] ||= Html2rss::Html::Navigator.extract_visible_text(node).to_s.scan(/\p{Alnum}+/).size
-          end
-
-          def date?(node)
-            @has_date[node] ||= begin
-              text = Html2rss::Html::Navigator.extract_visible_text(node).to_s
-              text.match?(%r{\b\d{4}[-/]\d{2}[-/]\d{2}\b}) ||
-                text.match?(/\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/i)
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/discovery/dom_clustering/overlap_resolver.rb

@@ -1,86 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    module Discovery
-      class DomClustering
-        ##
-        # Filters layout wrapper groups and resolves 1-to-1 nested card wrappers.
-        class OverlapResolver
-          # @param layout_tags [Set<String>] tags treated as layout containers
-          # @param word_counter [#avg_words] averages visible word counts for nodes
-          def initialize(layout_tags:, word_counter:)
-            @layout_tags = layout_tags
-            @word_counter = word_counter
-          end
-
-          # Discard group A if any node of A contains > 1 node of another group B
-          #
-          # @param groups [Hash{String => Array<Nokogiri::XML::Node>}] candidate groups
-          # @return [Hash{String => Array<Nokogiri::XML::Node>}] groups that are not layout wrappers
-          def filter_containers(groups)
-            groups.reject do |cls_a, nodes_a|
-              groups.any? { |cls_b, nodes_b| cls_a != cls_b && container_of?(nodes_a, nodes_b) }
-            end
-          end
-
-          # If group A contains group B with the same size, keep the real content card.
-          #
-          # @param groups [Hash{String => Array<Nokogiri::XML::Node>}] candidate groups
-          # @return [Hash{String => Array<Nokogiri::XML::Node>}] groups after 1-to-1 resolution
-          def filter_1_to_1_overlap(groups)
-            discarded = Set.new
-            groups.each_key do |cls_a|
-              groups.each_key do |cls_b|
-                next if cls_a == cls_b || discarded.include?(cls_a) || discarded.include?(cls_b)
-
-                resolve_1_to_1_overlap(cls_a, cls_b, groups, discarded)
-              end
-            end
-
-            groups.except(*discarded)
-          end
-
-          private
-
-          attr_reader :layout_tags, :word_counter
-
-          # rubocop:disable Metrics/MethodLength
-          def container_of?(nodes_a, nodes_b)
-            return false unless layout_tags.include?(nodes_b.first.name)
-
-            nodes_a.any? do |node_a|
-              count = 0
-              nodes_b.each do |node_b|
-                next if node_a == node_b
-
-                if Html2rss::Html::Navigator.descendant_of?(node_b, node_a)
-                  count += 1
-                  break if count > 1
-                end
-              end
-              count > 1
-            end
-          end
-          # rubocop:enable Metrics/MethodLength
-
-          def resolve_1_to_1_overlap(cls_a, cls_b, groups, discarded)
-            nodes_a = groups[cls_a]
-            nodes_b = groups[cls_b]
-            return if nodes_a.size != nodes_b.size
-
-            nested = nodes_a.zip(nodes_b).all? { |a, b| a != b && Html2rss::Html::Navigator.descendant_of?(b, a) }
-            return unless nested
-
-            discarded << (keep_descendant?(nodes_a, nodes_b) ? cls_a : cls_b)
-          end
-
-          def keep_descendant?(nodes_a, nodes_b)
-            word_counter.avg_words(nodes_b) >= 0.8 * word_counter.avg_words(nodes_a) &&
-              layout_tags.include?(nodes_b.first.name)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/discovery/dom_clustering.rb

@@ -1,119 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    module Discovery
-      ##
-      # Discovers repeated content card nodes in anchorless or classless DOM trees
-      # by evaluating class groups first and falling back to 1-level tag structure signatures.
-      class DomClustering
-        # Node tags considered layout containers
-        LAYOUT_TAG_NAMES = Set['div', 'section', 'article', 'li', 'ul', 'ol'].freeze
-        # HTML/layout tags excluded from candidate nodes (owned by Html::Navigator).
-        EXCLUDED_TAGS = Html2rss::Html::Navigator::CLUSTER_EXCLUDED_TAGS
-
-        class << self
-          ##
-          # Clusters elements in parsed_body and returns the best set of content card nodes.
-          #
-          # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
-          # @param minimum_selector_frequency [Integer] minimum frequency for candidate groups
-          # @return [Array<Nokogiri::XML::Node>] candidate nodes of the top-scoring cluster
-          def call(parsed_body, minimum_selector_frequency:)
-            new(parsed_body, minimum_frequency: minimum_selector_frequency).call
-          end
-        end
-
-        # @param parsed_body [Nokogiri::HTML::Document]
-        # @param minimum_frequency [Integer]
-        def initialize(parsed_body, minimum_frequency:)
-          @parsed_body = parsed_body
-          @minimum_frequency = minimum_frequency
-          @cache = {}.compare_by_identity
-        end
-
-        # @return [Array<Nokogiri::XML::Node>]
-        def call
-          class_result = cluster_by_class
-          return class_result unless class_result.empty?
-
-          cluster_by_structure
-        end
-
-        private
-
-        attr_reader :parsed_body, :minimum_frequency, :cache
-
-        def cluster_by_class
-          class_groups = collect_class_groups
-          return [] if class_groups.empty?
-
-          score_groups(class_groups)
-        end
-
-        def cluster_by_structure
-          structure_groups = collect_structure_groups
-          return [] if structure_groups.empty?
-
-          score_groups(structure_groups)
-        end
-
-        def score_groups(candidate_groups)
-          scorer = GroupScorer.new
-          resolver = OverlapResolver.new(layout_tags: LAYOUT_TAG_NAMES, word_counter: scorer)
-
-          non_containers = resolver.filter_containers(candidate_groups)
-          final_groups = resolver.filter_1_to_1_overlap(non_containers)
-
-          scorer.select_best_group(final_groups)
-        end
-
-        def collect_class_groups
-          groups = Hash.new { |h, k| h[k] = [] }
-          parsed_body.css('[class]').each { |node| add_class_node(node, groups) }
-          groups.select { |_, nodes| nodes.size >= minimum_frequency }
-        end
-
-        def add_class_node(node, groups)
-          return if EXCLUDED_TAGS.include?(node.name)
-          return if Html2rss::Html::Navigator.ignored_container_path?(node, cache)
-
-          cls = normalize_class(node['class'])
-          groups[cls] << node unless cls.empty?
-        end
-
-        def collect_structure_groups
-          groups = Hash.new { |h, k| h[k] = [] }
-          parsed_body.xpath('//*').each { |node| add_structure_node(node, groups) }
-          groups.select { |_, nodes| nodes.size >= minimum_frequency }
-        end
-
-        def add_structure_node(node, groups)
-          return if EXCLUDED_TAGS.include?(node.name)
-          return if Html2rss::Html::Navigator.ignored_container_path?(node, cache)
-
-          sig = structure_signature(node)
-          groups[sig] << node unless sig.empty?
-        end
-
-        def normalize_class(class_attr)
-          class_str = class_attr.to_s.strip
-          return '' if class_str.empty?
-
-          if class_str.include?(' ')
-            class_str.split(/\s+/).sort.join(' ')
-          else
-            class_str
-          end
-        end
-
-        def structure_signature(node)
-          child_elements = node.element_children
-          return '' if child_elements.empty?
-
-          child_elements.map(&:name).join('>')
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/discovery/list_candidates.rb

@@ -1,94 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    module Discovery
-      ##
-      # Builds repeated-list article container candidates from generic HTML.
-      class ListCandidates
-        ##
-        # Simplify an XPath selector by removing index notation.
-        #
-        # @param xpath [String] original XPath
-        # @return [String] XPath without positional indexes
-        def self.simplify_xpath(xpath)
-          xpath.gsub(/\[\d+\]/, '')
-        end
-
-        # @param parsed_body [Nokogiri::HTML::Document] parsed document
-        # @param minimum_selector_frequency [Integer] minimum repeated anchor path count
-        # @param use_top_selectors [Integer] number of frequent anchor paths to inspect
-        def initialize(parsed_body, minimum_selector_frequency:, use_top_selectors:)
-          @parsed_body = parsed_body
-          @minimum_selector_frequency = minimum_selector_frequency
-          @use_top_selectors = use_top_selectors
-        end
-
-        ##
-        # @param anchor_filter [#call] predicate for scraper-specific anchor eligibility
-        # @param boundary_condition [#call] predicate for article container boundary
-        # @yieldparam article_tag [Nokogiri::XML::Node] candidate article container
-        # @yieldparam selected_anchor [Nokogiri::XML::Node] anchor that made the container eligible
-        # @return [Enumerator]
-        def each_article_tag(anchor_filter:, boundary_condition:)
-          return enum_for(:each_article_tag, anchor_filter:, boundary_condition:) unless block_given?
-
-          article_tags(anchor_filter:, boundary_condition:).each { yield _1[:article_tag], _1[:selected_anchor] }
-        end
-
-        private
-
-        attr_reader :parsed_body, :minimum_selector_frequency, :use_top_selectors
-
-        def article_tags(anchor_filter:, boundary_condition:)
-          cache = {}.compare_by_identity
-          selectors(anchor_filter:).flat_map do |selector|
-            article_tags_for_selector(selector, boundary_condition, cache)
-          end
-        end
-
-        def article_tags_for_selector(selector, boundary_condition, cache = {}.compare_by_identity)
-          parsed_body.xpath(selector).filter_map do |selected_tag|
-            next if Html2rss::Html::Navigator.ignored_container_path?(selected_tag, cache)
-
-            article_tag = Html2rss::Html::Navigator.parent_until_condition(selected_tag, boundary_condition)
-            next unless article_tag
-
-            { article_tag:, selected_anchor: selected_tag }
-          end
-        end
-
-        def selectors(anchor_filter:)
-          anchor_counts(anchor_filter:)
-            .select { |_selector, count| count >= minimum_selector_frequency }
-            .max_by(use_top_selectors, &:last)
-            .map(&:first)
-        end
-
-        def anchor_counts(anchor_filter:)
-          cache = {}.compare_by_identity
-          Hash.new(0).tap do |counts|
-            each_anchor(anchor_filter:) do |node|
-              next if Html2rss::Html::Navigator.ignored_container_path?(node, cache)
-
-              path = self.class.simplify_xpath(node.path)
-              counts[path] += 1
-            end
-          end
-        end
-
-        def each_anchor(anchor_filter:)
-          return enum_for(:each_anchor, anchor_filter:) unless block_given?
-
-          traversal_root&.css(Html2rss::Html::Navigator::MAIN_ANCHOR_SELECTOR)&.each do |node|
-            yield node if anchor_filter.call(node)
-          end
-        end
-
-        def traversal_root
-          parsed_body.at_css('body, html') || parsed_body.root
-        end
-      end
-    end
-  end
-end