html2rss 0.32.3 → 0.33.0

data/lib/html2rss/auto_source/segmenter/list.rb

@@ -7,118 +7,86 @@ module Html2rss
       # Builds repeated-list article segments by grouping cards under a shared parent.
       # When the document has a +main+ or +[role=main]+, anchors outside it are dropped.
       module List
-        module_function
-
         ##
         # @param segmenter [Segmenter]
         # @return [Array<Segment>]
-        def call(segmenter)
-          primary = PrimaryLink.new(segmenter)
-          pairs = article_pairs(segmenter)
-          seen = {}.compare_by_identity
-
-          pairs.each_with_index.filter_map do |(article_tag, selected_anchor), position|
-            next unless seen[article_tag].nil?
-
-            seen[article_tag] = true
-
-            link = selected_anchor || primary.select(article_tag)
-            next unless link || segmenter.permit_unanchored
-
+        def self.call(segmenter)
+          article_pairs(segmenter).each_with_index.map do |(article_tag, link), position|
             Segment.build(root_node: article_tag, primary_link: link, strategy: :list, position:)
           end
         end
 
-        def article_pairs(segmenter)
+        def self.article_pairs(segmenter)
           top_groups(segmenter).flatten(1)
         end
-        module_function :article_pairs
         private_class_method :article_pairs
 
-        def top_groups(segmenter)
+        def self.top_groups(segmenter)
           cards_by_parent(segmenter)
             .select { |_parent, cards| cards.size >= segmenter.minimum_selector_frequency }
             .max_by(segmenter.use_top_selectors) { |_parent, cards| cards.size }
             .map(&:last)
         end
-        module_function :top_groups
         private_class_method :top_groups
 
-        def cards_by_parent(segmenter) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        def self.cards_by_parent(segmenter)
           index = segmenter.index
           main = main_landmark(index)
           groups = {}.compare_by_identity
 
-          index.each_node do |node|
-            next unless node.link?
-            next if index.ignored_chrome?(node)
-            next unless relevant_anchor?(segmenter, node)
-            next if main && !inside_main?(index, node, main)
-
+          index.each_node.select { |node| card_anchor?(segmenter, index, main, node) }.each do |node|
             card = parent_until_boundary(segmenter, node)
-            next unless card
-
-            parent = index.parent_of(card)
-            next unless parent
-
-            (groups[parent] ||= []) << [card, node]
+            (groups[index.parent_of(card)] ||= []) << [card, node]
           end
           groups
         end
-        module_function :cards_by_parent
         private_class_method :cards_by_parent
 
-        def main_landmark(index)
+        def self.card_anchor?(segmenter, index, main, node)
+          node.link? && !index.ignored_chrome?(node) && relevant_anchor?(segmenter, node) &&
+            (main.nil? || inside_main?(index, node, main))
+        end
+        private_class_method :card_anchor?
+
+        def self.main_landmark(index)
           index.each_node.find { |node| node.name == :main || node.attrs.raw['role'] == 'main' }
         end
-        module_function :main_landmark
         private_class_method :main_landmark
 
-        def inside_main?(index, node, main)
+        def self.inside_main?(index, node, main)
           node.equal?(main) || index.descendant_of?(node, main)
         end
-        module_function :inside_main?
         private_class_method :inside_main?
 
-        def relevant_anchor?(segmenter, node)
-          facts = segmenter.link_resolver.destination_facts(node)
-          return false unless facts
-
-          text = node.visible_text.to_s.strip
+        def self.relevant_anchor?(segmenter, node)
           !segmenter.noise_policy.noise_anchor?(
-            text:, destination_facts: facts, anchor: node, utility_landmark_ancestor: false
+            text: node.visible_text, destination_facts: segmenter.link_resolver.destination_facts(node),
+            anchor: node
           )
         end
-        module_function :relevant_anchor?
         private_class_method :relevant_anchor?
 
         BOUNDARY_TAGS = Set[:body, :html].freeze
         private_constant :BOUNDARY_TAGS
 
-        def parent_until_boundary(segmenter, node)
+        def self.parent_until_boundary(segmenter, node)
           index = segmenter.index
           link_counts = Hash.new { |hash, curr| hash[curr] = count_relevant_links(segmenter, curr) }
 
           index.parent_until(node, lambda { |curr|
             return true if BOUNDARY_TAGS.include?(curr.name)
-            return false if index.ignored_chrome?(curr)
 
-            parent = index.parent_of(curr)
-            parent && link_counts[parent] > link_counts[curr]
+            link_counts[index.parent_of(curr)] > link_counts[curr]
           })
         end
-        module_function :parent_until_boundary
         private_class_method :parent_until_boundary
 
-        def count_relevant_links(segmenter, node)
-          # Chrome inside a card must not stop the walk before the shared list parent.
-          if node.link?
-            relevant_anchor?(segmenter, node) ? 1 : 0
-          else
-            node.count_descendants { |child| child.link? && relevant_anchor?(segmenter, child) }
-          end
+        # Walk nodes are relevant anchors or their non-link ancestors, so a link counts as one.
+        def self.count_relevant_links(segmenter, node)
+          return 1 if node.link?
+
+          node.count_descendants { |child| child.link? && relevant_anchor?(segmenter, child) }
         end
-        module_function :count_relevant_links
         private_class_method :count_relevant_links
       end
     end

data/lib/html2rss/auto_source/segmenter/semantic.rb

@@ -9,12 +9,10 @@ module Html2rss
         # Tag names collected as leaf semantic containers.
         CANDIDATE_NAMES = %i[article section li tr div].to_set.freeze
 
-        module_function
-
         ##
         # @param segmenter [Segmenter]
         # @return [Array<Segment>]
-        def call(segmenter)
+        def self.call(segmenter)
           containers = collect_containers(segmenter.index)
           primary = PrimaryLink.new(segmenter)
 
@@ -26,7 +24,7 @@ module Html2rss
           end
         end
 
-        def collect_containers(index)
+        def self.collect_containers(index)
           non_leaf = mark_non_leaf_candidates(index.root)
           candidates = []
           index.each_node do |node|
@@ -40,14 +38,13 @@ module Html2rss
           candidates = filter_nested(candidates, index)
           sort_by_depth(candidates, index)
         end
-        module_function :collect_containers
         private_class_method :collect_containers
 
         TAG_MASKS = { article: 1, section: 2, li: 4, tr: 8, div: 16 }.freeze
         private_constant :TAG_MASKS
 
         # Single O(N) post-order pass: mark candidate nodes that contain a same-name candidate.
-        def mark_non_leaf_candidates(root) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        def self.mark_non_leaf_candidates(root) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
           non_leaf = Set.new.compare_by_identity
           visit = lambda do |node|
             seen_mask = 0
@@ -62,10 +59,9 @@ module Html2rss
           visit.call(root)
           non_leaf
         end
-        module_function :mark_non_leaf_candidates
         private_class_method :mark_non_leaf_candidates
 
-        def filter_nested(candidates, index)
+        def self.filter_nested(candidates, index)
           candidate_set = {}.compare_by_identity
           candidates.each { |c| candidate_set[c] = true }
           rejected = {}.compare_by_identity
@@ -78,25 +74,22 @@ module Html2rss
 
           candidates.reject { |c| rejected[c] }
         end
-        module_function :filter_nested
         private_class_method :filter_nested
 
-        def reject_ancestors(node, candidate_set, rejected, index)
+        def self.reject_ancestors(node, candidate_set, rejected, index)
           curr = index.parent_of(node)
           while curr && curr.name != :html
             rejected[curr] = true if candidate_set[curr]
             curr = index.parent_of(curr)
           end
         end
-        module_function :reject_ancestors
         private_class_method :reject_ancestors
 
-        def sort_by_depth(candidates, index)
+        def self.sort_by_depth(candidates, index)
           candidates.each_with_index
                     .sort_by { |node, i| [-index.depth_of(node), i] }
                     .map(&:first)
         end
-        module_function :sort_by_depth
         private_class_method :sort_by_depth
       end
     end

data/lib/html2rss/auto_source.rb

@@ -172,7 +172,7 @@ module Html2rss
         break if skip_html_padding?(tier, articles)
 
         if Scraper.heuristic_tier?(tier)
-          document ||= Scraper.normalize_sst(parsed_body)
+          document ||= SST::Normalizer.call(parsed_body)
           next unless document
 
           link_resolver ||= ::Html2rss::Scoring::LinkResolver.new(url)

data/lib/html2rss/batch.rb

@@ -10,17 +10,9 @@ module Html2rss
     # Default worker threads for batch operations.
     DEFAULT_CONCURRENCY = 5
 
+    # Outcome of a batch operation; +to_h+ is provided by Data.
     BatchResult = Data.define(:total, :successful, :results)
 
-    # Value object representing the outcome of a batch operation.
-    class BatchResult
-      ##
-      # @return [Hash{Symbol => Object}]
-      def to_h
-        { total:, successful:, results: }
-      end
-    end
-
     class << self
       ##
       # Executes a block concurrently across items using a bounded worker pool.
@@ -31,11 +23,8 @@ module Html2rss
       # @yieldparam item [Object]
       # @return [Array<Object>]
       def map(items, concurrency: DEFAULT_CONCURRENCY, &)
-        list = Array(items)
-        return [] if list.empty?
-
-        worker_count = worker_pool_size(list.size, concurrency)
-        process_queue(list, worker_count, &)
+        Option.integer(:concurrency, concurrency, min: 1, max: MAX_CONCURRENCY)
+        process_queue(Option.list(:items, items), [concurrency, items.size].min, &)
       end
 
       ##
@@ -46,11 +35,10 @@ module Html2rss
       # @yieldparam item [Object]
       # @return [BatchResult]
       def run(items, concurrency: DEFAULT_CONCURRENCY, &)
-        list = Array(items)
-        results = map(list, concurrency:, &)
+        results = map(items, concurrency:, &)
         successful = results.count { |entry| entry.is_a?(Hash) ? entry[:ok] : !entry.nil? }
 
-        BatchResult.new(total: list.size, successful:, results:)
+        BatchResult.new(total: items.size, successful:, results:)
       end
 
       ##
@@ -62,7 +50,7 @@ module Html2rss
       # @param items_selector [String, nil]
       # @return [Hash{Symbol => Object}] +:strategy+, +:items+, +:channel_title+, +:admission_drops+
       def scrape_wire(url:, strategy:, limit:, items_selector: nil) # rubocop:disable Metrics/MethodLength
-        plan = (strategy || :auto).to_sym
+        plan = Option.strategy(strategy)
         feed_args = { strategy: plan, limit: }
         feed_args[:items_selector] = items_selector unless items_selector.nil?
         feed_result = Html2rss.auto_feed_result(url, **feed_args)
@@ -84,7 +72,9 @@ module Html2rss
       # @param concurrency [Integer] number of worker threads (default 5, max 10)
       # @return [BatchResult]
       def batch_scrape(urls:, strategy: :auto, limit: 10, concurrency: DEFAULT_CONCURRENCY)
-        run(urls, concurrency:) { |url| scrape_single_url(url:, strategy:, limit:) }
+        strategy = Option.strategy(strategy)
+        Option.integer(:limit, limit, min: 1)
+        run(Option.list(:urls, urls), concurrency:) { |url| scrape_single_url(url:, strategy:, limit:) }
       end
 
       ##
@@ -95,7 +85,8 @@ module Html2rss
       # @param concurrency [Integer] number of worker threads (default 5, max 10)
       # @return [BatchResult]
       def batch_inspect(urls:, strategy: :auto, concurrency: DEFAULT_CONCURRENCY)
-        reports = PageRecon::Diagnostics.batch(urls:, strategy:, concurrency:)
+        reports = PageRecon::Diagnostics.batch(urls: Option.list(:urls, urls), strategy: Option.strategy(strategy),
+                                               concurrency:)
         results = reports.map(&:to_wire_h)
         successful = reports.count { |report| diagnostic_success?(report) }
 
@@ -111,16 +102,21 @@ module Html2rss
       # @option options [String, nil] :cache_dir optional HTML cache directory
       # @return [BatchResult]
       def batch_recon(urls:, strategy: :auto, concurrency: DEFAULT_CONCURRENCY, **)
-        results = Recon.batch(urls, strategy: (strategy || :auto).to_sym, max_threads: concurrency, **)
-        successful = results.count { |r| r.status ? r.status < 400 : false }
+        Option.integer(:concurrency, concurrency, min: 1, max: MAX_CONCURRENCY)
+        results = Recon.batch(Option.list(:urls, urls), strategy: Option.strategy(strategy),
+                                                        max_threads: concurrency, **)
+        successful = results.count { |result| http_success?(result.status) }
         BatchResult.new(total: results.size, successful:, results: results.map(&:to_h))
       end
 
       private
 
       def diagnostic_success?(report)
-        status = report.to_wire_h[:status]
-        status ? status < 400 : false
+        http_success?(report.to_wire_h[:status])
+      end
+
+      def http_success?(status)
+        !status.nil? && status < 400
       end
 
       def scrape_single_url(url:, strategy:, limit:)
@@ -129,16 +125,10 @@ module Html2rss
           url: url.to_s, ok: true, items_count: wire[:items].size,
           items: wire[:items], channel_title: wire[:channel_title]
         }
-      rescue StandardError => error
+      rescue Html2rss::Error => error
         { url: url.to_s, ok: false, error: error.message }
       end
 
-      def worker_pool_size(item_count, requested_concurrency)
-        return 1 if item_count <= 1
-
-        [[1, requested_concurrency.to_i].max, item_count, MAX_CONCURRENCY].min
-      end
-
       def process_queue(items, worker_count, &)
         queue = build_work_queue(items)
         results = Array.new(items.size)
@@ -157,7 +147,7 @@ module Html2rss
       end
 
       def drain_queue(queue, results)
-        until queue.empty?
+        loop do
           begin
             item, index = queue.pop(true)
           rescue ThreadError

data/lib/html2rss/capture/README.md

@@ -66,7 +66,7 @@ html2rss capture https://example.com --explain   # quality JSON on stderr; YAML
 ## Constraints
 
 - Selector quality depends on page structure; treat output as a first draft.
-- When HTML derivation misses (no articles, no SST document, fewer than `MIN_SELECTOR_MATCHES` pairs, or a rescued `ArgumentError`), Capture emits the default items selector with `segment_strategy: :default`. `enhance` still follows admission evidence (false when chrome drops are high). Non-HTML responses still omit selectors.
+- When HTML derivation misses (no articles, no SST document, fewer than `MIN_SELECTOR_MATCHES` pairs), Capture emits the default items selector with `segment_strategy: :default`. `enhance` still follows admission evidence (false when chrome drops are high). Non-HTML responses still omit selectors.
 - Botasaurus hops need `BOTASAURUS_SCRAPER_URL`.
 
 See also {Html2rss::AutoSource} for article discovery and {Html2rss::FeedPipeline} for the `:auto` request chain.

data/lib/html2rss/capture.rb

@@ -147,7 +147,6 @@ module Html2rss
 
       CaptureResult.new(
         config:,
-        yaml: "#{SCHEMA_MODELINE}\n#{Config.to_yaml(config)}",
         articles_count: outcome.articles.size,
         channel_title: ch_title,
         has_selectors: !selectors.empty?,
@@ -181,7 +180,7 @@ module Html2rss
         parsed_body: (response.parsed_body if response.html_response?),
         html: response.body
       )&.to_s
-    rescue StandardError
+    rescue Html2rss::Error
       nil
     end
 
@@ -270,9 +269,6 @@ module Html2rss
       return [*default_selectors, SelectorCandidates.empty] unless sst
 
       select_enhance_selectors(sst, articles)
-    rescue ArgumentError => error
-      Log.warn("Capture selector derivation failed: #{error.message}")
-      [*default_selectors, SelectorCandidates.empty]
     end
 
     def hint_selectors
@@ -338,22 +334,13 @@ module Html2rss
       end
     end
 
-    def find_matching_article(segment, articles_by_url) # rubocop:disable Metrics/AbcSize
-      href = segment.primary_link.attrs.href.to_s
-      return articles_by_url.values.find { |a| a.title == title_from_segment(segment) } if href.empty?
-
-      resolved = Url.from_relative(href, @url)
+    def find_matching_article(segment, articles_by_url)
+      resolved = Url.from_relative(segment.primary_link.attrs.href.to_s, @url)
       articles_by_url[resolved.to_s] || articles_by_url.values.find { |a| fuzzy_url_match?(a.url, resolved) }
     end
 
     def fuzzy_url_match?(article_url, resolved)
       article_url.to_s.end_with?(resolved.path.to_s)
-    rescue StandardError
-      false
-    end
-
-    def title_from_segment(segment)
-      segment.root_node.visible_text.to_s.strip
     end
 
     def lift_heading_link_roots(roots)
@@ -420,12 +407,7 @@ module Html2rss
     end
 
     def path_items_selector(roots)
-      paths = roots.map(&:tag_path)
-      common = common_path_prefix(paths)
-      tag_path = common.empty? ? paths.first.to_s : common
-      return if tag_path.empty?
-
-      css_from_trimmed_tag_path(tag_path)
+      css_from_trimmed_tag_path(common_path_prefix(roots.map(&:tag_path)))
     end
 
     def css_from_trimmed_tag_path(tag_path)
@@ -443,9 +425,7 @@ module Html2rss
       trimmed.empty? ? [segments.last] : trimmed
     end
 
-    def common_path_prefix(paths) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
-      return '' if paths.empty?
-
+    def common_path_prefix(paths)
       segments = paths.map { |p| p.split('/').reject(&:empty?) }
       prefix = segments.first
       segments.each do |seg|
@@ -466,7 +446,7 @@ module Html2rss
 
     def channel_title_from(response)
       Channel.from_response(response).title
-    rescue StandardError
+    rescue Html2rss::Error
       nil
     end
 
@@ -486,7 +466,7 @@ module Html2rss
       return final unless Url.from_absolute(@url).same_document?(Url.from_absolute(final))
 
       nil
-    rescue ArgumentError
+    rescue Html2rss::Url::InvalidUrlError
       nil
     end
   end