html2rss 0.24.0 → 0.26.0

data/lib/html2rss/selectors/post_processors/sanitize_html.rb

@@ -38,6 +38,15 @@ module Html2rss
       # Would return:
       #    '<p>Lorem <b>ipsum</b> dolor ...</p>'
       class SanitizeHtml < Base
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Sanitize HTML (sanitize gem RELAXED plus html2rss defaults: absolute URLs, ' \
+                      'safe link/img attributes, wrap lone images in anchors).'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'sanitize_html' }
+        ].freeze
+
         # @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy
         TAG_ATTRIBUTES = {
           'a' => {
@@ -81,6 +90,10 @@ module Html2rss
             'preload' => 'none'
           }
         }.freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :sanitize_html, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]
@@ -92,12 +105,19 @@ module Html2rss
         # @param html [String]
         # @param url [String, Html2rss::Url]
         # @return [String, nil]
+        # rubocop:disable ThreadSafety/ClassInstanceVariable
         def self.get(html, url)
           return nil if String(html).empty?
 
+          @fragment_cache ||= {}
+          key = [html, url.to_s]
+          return @fragment_cache[key] if @fragment_cache.key?(key)
+
+          @fragment_cache.clear if @fragment_cache.size > 256
           context = Selectors::Context.new(config: { channel: { url: } }, options: {})
-          new(html, context).get
+          @fragment_cache[key] = new(html, context).get
         end
+        # rubocop:enable ThreadSafety/ClassInstanceVariable
 
         ##
         # @param channel_url [String, Html2rss::Url]
@@ -117,6 +137,8 @@ module Html2rss
                 ->(env) { HtmlTransformers::WrapImgInA.new.call(**env) }
               ]
             )
+            config[:elements].delete('style')
+            config.delete(:css)
             config[:elements].push('audio', 'video', 'source')
             config.freeze
           end

data/lib/html2rss/selectors/post_processors/substring.rb

@@ -30,6 +30,27 @@ module Html2rss
       # Would return:
       #    'bar'
       class Substring < Base
+        # Required config field types (validator introspection via +Options+).
+        OPTION_TYPES = { start: Integer }.freeze
+
+        # Optional config field types (validated when the key is present and non-nil).
+        OPTIONAL_OPTION_TYPES = { end: Integer }.freeze
+
+        # Config fields required by this post-processor (validator / schema introspection).
+        Options = Struct.new(*OPTION_TYPES.keys, keyword_init: true)
+
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return a slice of the extracted string using Integer `start` and optional `end` ' \
+                      '(Ruby String#[] range semantics; end may be omitted).'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'substring', 'start' => 4, 'end' => 6 }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :substring, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/template.rb

@@ -34,6 +34,26 @@ module Html2rss
       # Would return:
       #    'Product (23,42€)'
       class Template < Base
+        # Required config field types (validator introspection via +Options+).
+        OPTION_TYPES = { string: String }.freeze
+
+        # Config fields required by this post-processor (validator / schema introspection).
+        Options = Struct.new(*OPTION_TYPES.keys, keyword_init: true)
+
+        # JSON Schema description exported via +schema_doc+.
+        # rubocop:disable Style/FormatStringToken -- documents Kernel#format `%{key}` placeholders
+        DESCRIPTION = 'Format a string with Kernel#format-style placeholders (`%{key}` / `%<key>s`). ' \
+                      '`%{self}` is the current selector value; other keys resolve sibling selectors.'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'template', 'string' => '`%{self}` (`%{price}`)' }
+        ].freeze
+        # rubocop:enable Style/FormatStringToken
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :template, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/schema_doc.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    ##
+    # Builds JSON Schema fragments from extractor / post-processor class constants.
+    #
+    # Each registry class owns +DESCRIPTION+, +EXAMPLES+, and optional +OPTION_TYPES+;
+    # this module is the export adapter only.
+    module SchemaDoc
+      # Maps Ruby option types to JSON Schema +type+ strings.
+      RUBY_TO_JSON_TYPE = {
+        String => 'string',
+        Integer => 'integer',
+        Hash => 'object',
+        Array => 'array'
+      }.freeze
+
+      module_function
+
+      ##
+      # @param name [Symbol, String] registry key
+      # @param klass [Class] post-processor class
+      # @return [Hash{Symbol => Object}] JSON Schema object for one post-processor
+      def for_post_processor(name:, klass:)
+        name = name.to_s
+        {
+          type: 'object',
+          title: name,
+          description: klass::DESCRIPTION,
+          examples: klass::EXAMPLES,
+          properties: post_processor_properties(name, klass),
+          required: post_processor_required(klass),
+          additionalProperties: true
+        }
+      end
+
+      ##
+      # @param name [Symbol, String] registry key
+      # @param klass [Class] extractor class
+      # @return [Hash{Symbol => Object}] JSON Schema for one extractor name
+      def for_extractor(name:, klass:)
+        name = name.to_s
+        {
+          type: 'string',
+          const: name,
+          title: name,
+          description: klass::DESCRIPTION,
+          examples: klass::EXAMPLES
+        }
+      end
+
+      ##
+      # @param name [String] registry key
+      # @param klass [Class]
+      # @return [Hash{Symbol => Hash}]
+      def post_processor_properties(name, klass)
+        properties = { name: { type: 'string', const: name } }
+        option_types_for(klass).each do |field, ruby_type|
+          properties[field] = { type: json_type_for(ruby_type) }
+        end
+        properties
+      end
+      module_function :post_processor_properties
+
+      ##
+      # @param klass [Class]
+      # @return [Array<String>]
+      def post_processor_required(klass)
+        required = ['name']
+        return required unless klass.const_defined?(:OPTION_TYPES)
+
+        required + klass::OPTION_TYPES.keys.map(&:to_s)
+      end
+      module_function :post_processor_required
+
+      ##
+      # @param klass [Class]
+      # @return [Hash{Symbol => Class}]
+      def option_types_for(klass)
+        types = {}
+        types.merge!(klass::OPTION_TYPES) if klass.const_defined?(:OPTION_TYPES)
+        types.merge!(klass::OPTIONAL_OPTION_TYPES) if klass.const_defined?(:OPTIONAL_OPTION_TYPES)
+        types
+      end
+      module_function :option_types_for
+
+      ##
+      # @param ruby_type [Class]
+      # @return [String]
+      def json_type_for(ruby_type)
+        RUBY_TO_JSON_TYPE.fetch(ruby_type) do
+          raise ArgumentError, "unsupported OPTION_TYPES mapping for #{ruby_type}"
+        end
+      end
+      module_function :json_type_for
+    end
+  end
+end

data/lib/html2rss/selectors.rb

@@ -48,8 +48,6 @@ module Html2rss
       @url = response.url
       @selectors = selectors
       @time_zone = time_zone
-
-      prepare_selectors!
       @rss_item_attributes = @selectors.keys.select { |key| SELECTABLE_SELECTOR_KEYS.include?(key) }
     end
 
@@ -180,24 +178,6 @@ module Html2rss
       )
     end
 
-    def prepare_selectors!
-      validate_url_and_link_exclusivity!
-      fix_url_and_link!
-    end
-
-    def validate_url_and_link_exclusivity!
-      return unless @selectors.key?(:url) && @selectors.key?(:link)
-
-      raise InvalidSelectorName, 'You must either use "url" or "link" your selectors. Using both is not supported.'
-    end
-
-    def fix_url_and_link!
-      return if @selectors[:url] || !@selectors.key?(:link)
-
-      @selectors = @selectors.dup
-      @selectors[:url] = @selectors[:link]
-    end
-
     def parsed_body
       parsed_body_for(response)
     end

data/lib/html2rss/sst/attrs.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Shared empty defaults for {Attrs} (kept outside Data.define for constant visibility).
+    module AttrDefaults
+      # Shared empty class_names array for Attrs defaults.
+      EMPTY_CLASS_NAMES = [].freeze
+      # Shared empty raw attribute hash for Attrs defaults.
+      EMPTY_RAW = {}.freeze
+    end
+
+    ##
+    # Typed HTML attributes for an SST node. Absent fields are +nil+.
+    Attrs = Data.define(
+      :href, :src, :id, :class_names, :datetime, :itemprop, :style, :srcset, :type, :raw
+    ) do
+      class << self
+        ##
+        # @param href [String, nil]
+        # @param src [String, nil]
+        # @param id [String, nil]
+        # @param class_names [Array<String>, nil]
+        # @param datetime [String, nil]
+        # @param itemprop [String, nil]
+        # @param style [String, nil]
+        # @param srcset [String, nil]
+        # @param type [String, nil]
+        # @param raw [Hash{String => String}, nil]
+        # @return [Attrs]
+        # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
+        def build(href: nil, src: nil, id: nil, class_names: nil, datetime: nil, itemprop: nil,
+                  style: nil, srcset: nil, type: nil, raw: nil)
+          new(
+            href: blank_to_nil(href),
+            src: blank_to_nil(src),
+            id: blank_to_nil(id),
+            class_names: normalize_class_names(class_names),
+            datetime: blank_to_nil(datetime),
+            itemprop: blank_to_nil(itemprop),
+            style: blank_to_nil(style),
+            srcset: blank_to_nil(srcset),
+            type: blank_to_nil(type),
+            raw: normalize_raw(raw)
+          )
+        end
+        # rubocop:enable Metrics/MethodLength, Metrics/ParameterLists
+
+        ##
+        # @return [Attrs]
+        def empty = EMPTY_ATTRS
+
+        private
+
+        def blank_to_nil(value)
+          str = value&.to_s
+          return if str.nil? || str.strip.empty?
+
+          str.strip.freeze
+        end
+
+        def normalize_class_names(class_names)
+          case class_names
+          when nil then AttrDefaults::EMPTY_CLASS_NAMES
+          when Array then class_names.map { |token| token.to_s.freeze }.freeze
+          else raise ArgumentError, "class_names must be an Array, got #{class_names.class}"
+          end
+        end
+
+        def normalize_raw(raw)
+          case raw
+          when nil then AttrDefaults::EMPTY_RAW
+          when Hash then raw.transform_values { |v| v.to_s.freeze }.freeze
+          else raise ArgumentError, "raw must be a Hash, got #{raw.class}"
+          end
+        end
+      end
+
+      ##
+      # @return [String]
+      def class_attr = class_names.join(' ')
+    end
+
+    # Shared empty Attrs instance for nodes without attributes.
+    EMPTY_ATTRS = Attrs.new(
+      href: nil, src: nil, id: nil, class_names: AttrDefaults::EMPTY_CLASS_NAMES, datetime: nil,
+      itemprop: nil, style: nil, srcset: nil, type: nil, raw: AttrDefaults::EMPTY_RAW
+    ).freeze
+  end
+end

data/lib/html2rss/sst/document.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Immutable SST document: root node plus relationship index.
+    Document = Data.define(:root, :index, :degraded, :node_count) do
+      ##
+      # @param root [Node]
+      # @param index [Index]
+      # @param degraded [Boolean] true when MAX_NODES forced semantic-tag-only mode
+      # @param node_count [Integer]
+      # @return [Document]
+      # @raise [ArgumentError] when root/index missing
+      def self.build(root:, index:, degraded: false, node_count: 0)
+        raise ArgumentError, 'root is required' unless root.is_a?(Node)
+        raise ArgumentError, 'index is required' unless index.is_a?(Index)
+
+        new(root:, index:, degraded: !!degraded, node_count: Integer(node_count))
+      end
+    end
+  end
+end

data/lib/html2rss/sst/index.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Parent/depth indices for an SST tree. Built once by the Normalizer so
+    # Scoring/Segmenter never need Nokogiri ancestor walks.
+    class Index
+      # Weak node→index bindings so +Node#visible_text+ can memoize without call-site churn.
+      NODE_BINDINGS = ObjectSpace::WeakMap.new
+
+      ##
+      # @param node [Node]
+      # @return [Index, nil]
+      def self.for_node(node) = NODE_BINDINGS[node]
+
+      ##
+      # @param root [Node]
+      # @param parents [Hash{Node => Node, nil}]
+      # @param depths [Hash{Node => Integer}]
+      # @param ignored_chrome [Hash{Node => Boolean}]
+      def initialize(root:, parents:, depths:, ignored_chrome:)
+        @root = root
+        @parents = parents
+        @depths = depths
+        @ignored_chrome = ignored_chrome
+        @visible_text = {}.compare_by_identity
+        @word_count = {}.compare_by_identity
+        bind_tree(root)
+      end
+
+      attr_reader :root
+
+      ##
+      # @param node [Node]
+      # @return [Node, nil]
+      def parent_of(node) = @parents[node]
+
+      ##
+      # @param node [Node]
+      # @return [Integer]
+      def depth_of(node) = @depths.fetch(node, 0)
+
+      ##
+      # @param node [Node]
+      # @return [Boolean]
+      def ignored_chrome?(node) = @ignored_chrome.fetch(node, false)
+
+      ##
+      # Memoized default visible text (separator space, no excludes).
+      #
+      # @param node [Node]
+      # @return [String, nil]
+      def memo_visible_text(node)
+        return @visible_text[node] if @visible_text.key?(node)
+
+        @visible_text[node] = Text.extract(node)
+      end
+
+      ##
+      # @param node [Node]
+      # @return [Integer]
+      def memo_word_count(node)
+        return @word_count[node] if @word_count.key?(node)
+
+        @word_count[node] = memo_visible_text(node).to_s.scan(/\p{Alnum}+/).size
+      end
+
+      ##
+      # @param child [Node]
+      # @param ancestor [Node]
+      # @return [Boolean]
+      def descendant_of?(child, ancestor)
+        curr = parent_of(child)
+        while curr
+          return true if curr.equal?(ancestor)
+
+          curr = parent_of(curr)
+        end
+        false
+      end
+
+      ##
+      # @param node [Node]
+      # @param condition [#call]
+      # @return [Node, nil]
+      def parent_until(node, condition)
+        curr = node
+        while curr && curr.name != :html
+          return curr if condition.call(curr)
+
+          curr = parent_of(curr)
+        end
+        nil
+      end
+
+      ##
+      # @yieldparam node [Node]
+      # @return [Enumerator]
+      def each_node(&)
+        root.each_node(&)
+      end
+
+      private
+
+      def bind_tree(node)
+        NODE_BINDINGS[node] = self
+        node.children.each { bind_tree(_1) }
+      end
+    end
+  end
+end

data/lib/html2rss/sst/node.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Immutable simplified semantic tree node. Predicates and traversal live on
+    # the type so Scoring/Segmenter never touch Nokogiri.
+    Node = Data.define(:name, :attrs, :own_text, :children, :tag_path) do
+      ##
+      # @param name [Symbol, String]
+      # @param attrs [Attrs, nil]
+      # @param own_text [String, nil]
+      # @param children [Array<Node>, nil]
+      # @param tag_path [String, nil] index-free ancestry path for list clustering
+      # @return [Node]
+      # @raise [ArgumentError] on invalid name/attrs/children
+      def self.build(name:, attrs: nil, own_text: nil, children: nil, tag_path: nil)
+        attrs_value = coerce_attrs(attrs)
+        kids = coerce_children(children)
+
+        new(
+          name: coerce_name(name),
+          attrs: attrs_value,
+          own_text: own_text.to_s.freeze,
+          children: kids,
+          tag_path: tag_path.to_s.freeze
+        )
+      end
+
+      def self.coerce_name(name)
+        raise ArgumentError, 'name is required' if name.nil? || name.to_s.strip.empty?
+
+        name.to_sym
+      end
+      private_class_method :coerce_name
+
+      def self.coerce_attrs(attrs)
+        attrs_value = attrs.nil? ? Attrs.empty : attrs
+        raise ArgumentError, 'attrs must be SST::Attrs' unless attrs_value.is_a?(Attrs)
+
+        attrs_value
+      end
+      private_class_method :coerce_attrs
+
+      def self.coerce_children(children)
+        kids = Array(children).freeze
+        raise ArgumentError, 'children must be SST::Node instances' unless kids.all?(Node)
+
+        kids
+      end
+      private_class_method :coerce_children
+
+      ##
+      # @return [Boolean]
+      def link?
+        name == :a && eligible_href?
+      end
+
+      ##
+      # @return [Boolean]
+      def image? = name == :img
+
+      ##
+      # @return [Boolean]
+      def heading? = Tags::HEADING_NAMES.include?(name)
+
+      ##
+      # @return [Boolean]
+      def utility_landmark? = Tags::UTILITY_LANDMARK_NAMES.include?(name)
+
+      ##
+      # @return [Boolean]
+      def ignored_container_name? = Tags::IGNORED_CONTAINER_NAMES.include?(name)
+
+      ##
+      # @return [Boolean]
+      def eligible_href?
+        href = attrs.href
+        return false if href.nil? || href.empty?
+
+        Tags::SKIP_HREF_PREFIXES.none? { |prefix| href.start_with?(prefix) }
+      end
+
+      ##
+      # Depth-first enumeration of self and descendants.
+      #
+      # @yieldparam node [Node]
+      # @return [Enumerator, Array<Node>]
+      def each_node(&block)
+        return enum_for(:each_node) unless block
+
+        yield self
+        children.each { |child| child.each_node(&block) }
+      end
+
+      ##
+      # @return [Array<Node>] all descendants excluding self
+      def descendants
+        children.flat_map { |child| [child, *child.descendants] }
+      end
+
+      ##
+      # @return [Array<Node>]
+      def find_all(&predicate)
+        each_node.select(&predicate)
+      end
+
+      ##
+      # @return [Node, nil]
+      def find(&predicate)
+        each_node.find(&predicate)
+      end
+
+      ##
+      # Visible text with block newlines, excluding optional subtrees.
+      #
+      # @param separator [String]
+      # @param exclude [Array<Node>, nil]
+      # @return [String, nil]
+      def visible_text(separator: ' ', exclude: nil)
+        if exclude.nil? && separator == ' ' && (index = Index.for_node(self))
+          return index.memo_visible_text(self)
+        end
+
+        Text.extract(self, separator:, exclude:)
+      end
+
+      ##
+      # @return [Integer] alphanumeric word count of visible text
+      def word_count
+        if (index = Index.for_node(self))
+          return index.memo_word_count(self)
+        end
+
+        visible_text.to_s.scan(/\p{Alnum}+/).size
+      end
+
+      ##
+      # @return [Float] words per descendant link
+      def text_density
+        words = word_count
+        links = descendants.count(&:link?)
+        links.zero? ? words.to_f : words.to_f / links
+      end
+    end
+  end
+end