html2rss 0.22.2 → 0.24.0

data/lib/html2rss/config/class_methods.rb

@@ -1,183 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class Config
-    ##
-    # Public class-level helpers for loading, validating, and exporting config.
-    module ClassMethods
-      # Sentinel to differentiate omitted params from explicit `nil`.
-      UNSET = Object.new.freeze
-
-      ##
-      # Returns the exported JSON Schema for html2rss configuration.
-      #
-      # @return [Hash{String => Object}] JSON Schema represented as a Ruby hash
-      def json_schema
-        Schema.json_schema
-      end
-
-      ##
-      # Returns the exported JSON Schema as JSON.
-      #
-      # @param pretty [Boolean] whether to pretty-print the JSON output
-      # @return [String] serialized JSON Schema
-      def json_schema_json(pretty: true)
-        pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
-      end
-
-      ##
-      # Validates a configuration hash with the runtime validator.
-      #
-      # @param config [Hash{Symbol => Object}] the configuration hash
-      # @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting
-      # @return [Dry::Validation::Result] validation result after defaults and deprecations are applied
-      def validate(config, params: UNSET)
-        prepared_config = prepare_for_validation(resolve_effective_config(config, params:))
-
-        Validator.new.call(prepared_config)
-      rescue DynamicParams::ParamsMissing => error
-        prepared_config = prepare_for_validation(HashUtil.deep_symbolize_keys(config, context: 'config'))
-        prepared_config[:dynamic_params_error] = error.message
-
-        Validator.new.call(prepared_config)
-      end
-
-      ##
-      # Returns the packaged JSON Schema file path.
-      #
-      # @return [String] absolute path to the packaged JSON Schema file
-      def schema_path
-        Schema.path
-      end
-
-      ##
-      # Loads and validates a YAML configuration file.
-      #
-      # @param file [String] the YAML file to load
-      # @param feed_name [String, nil] optional feed name for multi-feed files
-      # @param multiple_feeds_key [Symbol] key under which multiple feeds are defined
-      # @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting
-      # @return [Dry::Validation::Result] validation result after defaults and deprecations are applied
-      def validate_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS, params: UNSET)
-        validate(load_yaml(file, feed_name, multiple_feeds_key:), params:)
-      end
-
-      ##
-      # Loads the feed configuration from a YAML file.
-      #
-      # Supports multiple feeds defined under the specified key (default :feeds).
-      #
-      # @param file [String] the YAML file to load.
-      # @param feed_name [String, nil] the feed name when using multiple feeds.
-      # @param multiple_feeds_key [Symbol] the key under which multiple feeds are defined.
-      # @return [Hash{Symbol => Object}] the configuration hash.
-      # @raise [ArgumentError] if the file doesn't exist or feed is not found.
-      # rubocop:disable Metrics/MethodLength
-      def load_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS)
-        raise ArgumentError, "File '#{file}' does not exist" unless File.exist?(file)
-        raise ArgumentError, "`#{multiple_feeds_key}` is a reserved feed name" if feed_name == multiple_feeds_key
-
-        yaml = YAML.safe_load_file(file, symbolize_names: true)
-
-        return yaml unless yaml.key?(multiple_feeds_key)
-
-        unless feed_name
-          available_feeds = yaml.fetch(multiple_feeds_key).keys.join(', ')
-          raise ArgumentError,
-                "Feed name is required under `#{multiple_feeds_key}`. Available feeds: #{available_feeds}"
-        end
-
-        config = yaml.dig(multiple_feeds_key, feed_name.to_sym)
-        raise ArgumentError, "Feed '#{feed_name}' not found under `#{multiple_feeds_key}` key." unless config
-
-        MultipleFeedsConfig.to_single_feed(config, yaml, multiple_feeds_key:)
-      end
-      # rubocop:enable Metrics/MethodLength
-
-      ##
-      # Processes the provided configuration hash, applying dynamic parameters if given,
-      # and returns a new configuration object.
-      #
-      # @param config [Hash{Symbol => Object}] the configuration hash.
-      # @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting.
-      # @return [Html2rss::Config] the configuration object.
-      def from_hash(config, params: UNSET)
-        new(resolve_effective_config(config, params:))
-      end
-
-      ##
-      # Builds a top-level auto-source feed config for the public shortcut APIs.
-      #
-      # @param url [String] source page URL
-      # @param items_selector [String, nil] optional selector hint for item extraction
-      # @param request_controls [Html2rss::RequestControls, nil] explicit request controls to write
-      # @return [Hash{Symbol => Object}] feed config hash ready for {from_hash}
-      def auto_source_config(url:, items_selector: nil, request_controls: nil)
-        config = {
-          channel: default_config[:channel].merge(url:),
-          auto_source: AutoSource::DEFAULT_CONFIG
-        }
-
-        request_controls ||= Html2rss::RequestControls.new
-        request_controls.apply_to(config)
-
-        config[:selectors] = { items: { selector: items_selector, enhance: true } } if items_selector
-        config
-      end
-
-      ##
-      # Provides a default configuration.
-      #
-      # @return [Hash{Symbol => Object}] a hash with default configuration values.
-      def default_config
-        {
-          strategy: default_strategy_name,
-          request: default_request_config,
-          channel: { time_zone: 'UTC' },
-          headers: RequestHeaders.browser_defaults,
-          stylesheets: Html2rss.configuration.stylesheets || []
-        }
-      end
-
-      # @return [Symbol] the default strategy for feed orchestration
-      def default_strategy_name
-        Html2rss.configuration.default_strategy || :auto
-      end
-
-      private
-
-      def default_request_config
-        {
-          max_redirects: RequestService::Policy::DEFAULTS[:max_redirects],
-          max_requests: RequestService::Policy::DEFAULTS[:max_requests],
-          total_timeout_seconds: RequestService::Policy::DEFAULTS[:total_timeout_seconds]
-        }
-      end
-
-      def resolve_effective_config(config, params:)
-        effective_config = HashUtil.deep_symbolize_keys(config, context: 'config')
-        resolved_params = parameter_defaults(effective_config)
-        unless params.equal?(UNSET) || params.nil?
-          resolved_params.merge!(HashUtil.deep_symbolize_keys(params, context: 'params'))
-        end
-
-        effective_config[:headers] = DynamicParams.call(effective_config[:headers], resolved_params)
-        effective_config[:channel] = DynamicParams.call(effective_config[:channel], resolved_params)
-
-        effective_config
-      end
-
-      def parameter_defaults(config)
-        config.fetch(:parameters, {})
-              .filter_map do |name, definition|
-                [name, definition[:default]] if definition.is_a?(Hash) && definition.key?(:default)
-              end
-              .to_h
-      end
-
-      def prepare_for_validation(config)
-        Config::Preparer.new.call(HashUtil.deep_dup(config))
-      end
-    end
-  end
-end

data/lib/html2rss/html_extractor/date_extractor.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class HtmlExtractor
-    # Extracts the earliest date from an article_tag.
-    class DateExtractor
-      # @param article_tag [Nokogiri::XML::Element] article container node
-      # @return [DateTime, nil]
-      def self.call(article_tag)
-        times = article_tag.css('[datetime]').filter_map do |tag|
-          DateTime.parse(tag['datetime'])
-        rescue ArgumentError, TypeError
-          nil
-        end
-
-        times.min
-      end
-    end
-  end
-end

data/lib/html2rss/html_extractor/enclosure_extractor.rb

@@ -1,91 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class HtmlExtractor
-    ##
-    # Extracts enclosures from HTML tags using various strategies.
-    class EnclosureExtractor
-      # CSS union query covering images, media, PDFs, iframes, and archives.
-      SELECTOR = [
-        'img[src]:not([src^="data"])',
-        'video source[src]',
-        'audio source[src]',
-        'audio[src]',
-        'a[href$=".pdf"]',
-        'iframe[src]',
-        'a[href$=".zip"]',
-        'a[href$=".tar.gz"]',
-        'a[href$=".tgz"]'
-      ].join(',').freeze
-
-      # @param article_tag [Nokogiri::XML::Element] article container node
-      # @param base_url [String, Html2rss::Url] base URL for relative enclosure links
-      # @return [Array<Hash{Symbol => Object}>] normalized enclosure hashes
-      def self.call(article_tag, base_url)
-        article_tag.css(SELECTOR).filter_map do |element|
-          extract_from_element(element, base_url)
-        end
-      end
-
-      def self.extract_from_element(element, base_url)
-        case element.name
-        when 'img'
-          extract_image(element, base_url)
-        when 'video', 'audio', 'source'
-          extract_media(element, base_url)
-        when 'iframe'
-          extract_iframe(element, base_url)
-        when 'a'
-          extract_a(element, base_url)
-        end
-      end
-
-      def self.extract_image(img, base_url)
-        src = img['src'].to_s
-        return if src.empty?
-
-        abs_url = Url.from_relative(src, base_url)
-        {
-          url: abs_url,
-          type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'image/jpeg')
-        }
-      end
-
-      def self.extract_media(element, base_url)
-        src = element['src'].to_s
-        return if src.empty?
-
-        {
-          url: Url.from_relative(src, base_url),
-          type: element['type']
-        }
-      end
-
-      def self.extract_iframe(iframe, base_url)
-        src = iframe['src'].to_s
-        return if src.empty?
-
-        abs_url = Url.from_relative(src, base_url)
-        {
-          url: abs_url,
-          type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'text/html')
-        }
-      end
-
-      def self.extract_a(link, base_url)
-        href = link['href'].to_s
-        return if href.empty?
-
-        abs_url = Url.from_relative(href, base_url)
-
-        if href.end_with?('.pdf')
-          { url: abs_url, type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url) }
-        else
-          { url: abs_url, type: 'application/zip' }
-        end
-      end
-
-      private_class_method :extract_from_element, :extract_image, :extract_media, :extract_iframe, :extract_a
-    end
-  end
-end

data/lib/html2rss/html_extractor/heading_extractor.rb

@@ -1,50 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class HtmlExtractor
-    ##
-    # HeadingExtractor identifies and returns the best heading element within a container.
-    class HeadingExtractor
-      # Heading tags used to prioritize title extraction.
-      HEADING_TAGS = HtmlExtractor::HEADING_TAGS
-
-      class << self
-        ##
-        # @param article_tag [Nokogiri::XML::Element] container node
-        # @param fallback_anchorless [Boolean] whether to use fallback search
-        # @param selected_anchor [Nokogiri::XML::Node, nil] anchor element
-        # @return [Nokogiri::XML::Node, nil] the heading node, if found
-        def call(article_tag, fallback_anchorless:, selected_anchor:)
-          tags = article_tag.css(HEADING_TAGS.join(','))
-          if tags.any?
-            select_best_heading(tags)
-          elsif fallback_anchorless && selected_anchor.nil?
-            fallback_heading(article_tag)
-          end
-        end
-
-        private
-
-        def select_best_heading(tags)
-          min_tag_name = tags.map(&:name).min
-          best_tag = nil
-          max_size = -1
-
-          tags.each do |tag|
-            next if tag.name != min_tag_name
-
-            size = TextExtractor.call(tag)&.size.to_i
-            (best_tag = tag) && (max_size = size) if size > max_size
-          end
-
-          best_tag
-        end
-
-        def fallback_heading(article_tag)
-          fallback_tags = article_tag.css('strong, b, [class*="title"], [class*="font-bold"], [class*="font-semibold"]')
-          fallback_tags.find { |t| !TextExtractor.call(t).to_s.strip.empty? }
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/html_extractor/id_generator.rb

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-require 'zlib'
-
-module Html2rss
-  class HtmlExtractor
-    ##
-    # IdGenerator determines the unique ID for an article container node.
-    class IdGenerator
-      class << self
-        ##
-        # @param article_tag [Nokogiri::XML::Element] container node
-        # @param heading [Nokogiri::XML::Node, nil] heading node
-        # @param url [Html2rss::Url, nil] absolute article URL
-        # @param selected_anchor [Nokogiri::XML::Node, nil] anchor element
-        # @param fallback_anchorless [Boolean] whether to use fallback hashing
-        # @return [String, nil] the generated ID, if any
-        def call(article_tag, heading:, url:, selected_anchor:, fallback_anchorless:)
-          id_from_dom = parse_id_from_dom(article_tag, url, selected_anchor)
-          return id_from_dom if id_from_dom
-
-          heading_text = resolve_heading_text(article_tag, heading, fallback_anchorless)
-          if heading_text && !heading_text.strip.empty?
-            generate_slug(heading_text)
-          elsif fallback_anchorless
-            generate_content_hash(article_tag)
-          end
-        end
-
-        private
-
-        def parse_id_from_dom(article_tag, url, selected_anchor)
-          candidates = [article_tag['id'], article_tag.at_css('[id]')&.attr('id')]
-          candidates += [url&.path, url&.query] if selected_anchor
-          candidates.compact.reject(&:empty?).first
-        end
-
-        def resolve_heading_text(article_tag, heading, fallback_anchorless)
-          text = heading ? TextExtractor.call(heading) : nil
-          if text.nil? || text.strip.empty?
-            fallback_text_node_content(article_tag, fallback_anchorless)
-          else
-            text
-          end
-        end
-
-        def fallback_text_node_content(article_tag, fallback_anchorless)
-          return unless fallback_anchorless
-
-          article_tag.xpath('.//text()').find { |t| !t.text.strip.empty? }&.text&.strip
-        end
-
-        def generate_slug(text)
-          slug = text.downcase.gsub(/[^a-z0-9]+/, '-')
-          slug = slug[1..] if slug.start_with?('-')
-          slug = slug[0..-2] if slug.end_with?('-')
-          slug unless slug.empty?
-        end
-
-        def generate_content_hash(article_tag)
-          text = TextExtractor.call(article_tag).to_s.strip
-          Zlib.crc32(text).to_s(36) unless text.empty?
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/html_extractor/image_extractor.rb

@@ -1,58 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class HtmlExtractor
-    ##
-    # Image is responsible for extracting image URLs the article_tag.
-    class ImageExtractor
-      # @param article_tag [Nokogiri::XML::Element] article container node
-      # @param base_url [String, Html2rss::Url] base URL for relative image URLs
-      # @return [Html2rss::Url, nil] best candidate image URL
-      def self.call(article_tag, base_url:)
-        img_src = from_source(article_tag) ||
-                  from_img(article_tag) ||
-                  from_style(article_tag)
-
-        Url.from_relative(img_src, base_url) if img_src
-      end
-
-      # @param article_tag [Nokogiri::XML::Element] article container node
-      # @return [String, nil] src attribute from first matching image tag
-      def self.from_img(article_tag)
-        article_tag.at_css('img[src]:not([src^="data"])')&.[]('src')
-      end
-
-      ##
-      # Extracts the largest image source from the srcset attribute
-      # of an img tag or a source tag inside a picture tag.
-      #
-      # @param article_tag [Nokogiri::XML::Element] article container node
-      # @return [String, nil] largest srcset URL candidate
-      # @see <https://developer.mozilla.org/en-US/docs/Learn/HTML/Multimedia_and_embedding/Responsive_images>
-      # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/img#srcset>
-      # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/picture>
-      def self.from_source(article_tag) # rubocop:disable Metrics/AbcSize
-        hash = article_tag.css('img[srcset], picture > source[srcset]').flat_map do |source|
-          source['srcset'].to_s.scan(/(\S+)\s+(\d+w|\d+h)[\s,]?/).map do |url, width|
-            next if url.nil? || url.start_with?('data:')
-
-            width_value = width.to_i.zero? ? 0 : width.scan(/\d+/).first.to_i
-
-            [width_value, url.strip]
-          end
-        end.compact.to_h
-
-        hash[hash.keys.max]
-      end
-
-      # @param article_tag [Nokogiri::XML::Element] article container node
-      # @return [String, nil] best style-based background image URL
-      def self.from_style(article_tag)
-        article_tag.css('[style*="url"]')
-                   .filter_map { |tag| tag['style'][/url\(['"]?(.*?)['"]?\)/, 1] }
-                   .reject { |src| src.start_with?('data:') }
-                   .max_by(&:size)
-      end
-    end
-  end
-end

data/lib/html2rss/html_extractor/list_candidates.rb

@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class HtmlExtractor
-    ##
-    # Builds repeated-list article container candidates from generic HTML.
-    class ListCandidates
-      ##
-      # Simplify an XPath selector by removing index notation.
-      #
-      # @param xpath [String] original XPath
-      # @return [String] XPath without positional indexes
-      def self.simplify_xpath(xpath)
-        xpath.gsub(/\[\d+\]/, '')
-      end
-
-      # @param parsed_body [Nokogiri::HTML::Document] parsed document
-      # @param minimum_selector_frequency [Integer] minimum repeated anchor path count
-      # @param use_top_selectors [Integer] number of frequent anchor paths to inspect
-      def initialize(parsed_body, minimum_selector_frequency:, use_top_selectors:)
-        @parsed_body = parsed_body
-        @minimum_selector_frequency = minimum_selector_frequency
-        @use_top_selectors = use_top_selectors
-      end
-
-      ##
-      # @param anchor_filter [#call] predicate for scraper-specific anchor eligibility
-      # @param boundary_condition [#call] predicate for article container boundary
-      # @yieldparam article_tag [Nokogiri::XML::Node] candidate article container
-      # @yieldparam selected_anchor [Nokogiri::XML::Node] anchor that made the container eligible
-      # @return [Enumerator]
-      def each_article_tag(anchor_filter:, boundary_condition:)
-        return enum_for(:each_article_tag, anchor_filter:, boundary_condition:) unless block_given?
-
-        article_tags(anchor_filter:, boundary_condition:).each { yield _1[:article_tag], _1[:selected_anchor] }
-      end
-
-      private
-
-      attr_reader :parsed_body, :minimum_selector_frequency, :use_top_selectors
-
-      def article_tags(anchor_filter:, boundary_condition:)
-        selectors(anchor_filter:).flat_map do |selector|
-          article_tags_for_selector(selector, boundary_condition)
-        end
-      end
-
-      def article_tags_for_selector(selector, boundary_condition)
-        parsed_body.xpath(selector).filter_map do |selected_tag|
-          next if HtmlExtractor.ignored_container_path?(selected_tag)
-
-          article_tag = HtmlNavigator.parent_until_condition(selected_tag, boundary_condition)
-          next unless article_tag
-
-          { article_tag:, selected_anchor: selected_tag }
-        end
-      end
-
-      def selectors(anchor_filter:)
-        anchor_counts(anchor_filter:)
-          .select { |_selector, count| count >= minimum_selector_frequency }
-          .max_by(use_top_selectors, &:last)
-          .map(&:first)
-      end
-
-      def anchor_counts(anchor_filter:)
-        Hash.new(0).tap do |counts|
-          each_anchor(anchor_filter:) do |node|
-            path = self.class.simplify_xpath(node.path)
-            counts[path] += 1 unless HtmlExtractor.ignored_container_path?(path)
-          end
-        end
-      end
-
-      def each_anchor(anchor_filter:)
-        return enum_for(:each_anchor, anchor_filter:) unless block_given?
-
-        traversal_root&.css(HtmlExtractor::MAIN_ANCHOR_SELECTOR)&.each do |node|
-          yield node if anchor_filter.call(node)
-        end
-      end
-
-      def traversal_root
-        parsed_body.at_css('body, html') || parsed_body.root
-      end
-    end
-  end
-end