html2rss 0.23.0 → 0.24.0

data/lib/html2rss/selectors.rb

@@ -31,6 +31,11 @@ module Html2rss
     ITEM_TAGS = %i[title url description author comments published_at guid enclosure categories].freeze
     # Item attributes that require dedicated extraction logic.
     SPECIAL_ATTRIBUTES = Set[:guid, :enclosure, :categories].freeze
+    # Config selector keys that map onto a different {Article} attribute.
+    # +:enclosure+ stays singular in YAML; Article stores +:enclosures+.
+    SELECTOR_TO_ARTICLE_KEY = { enclosure: :enclosures }.freeze
+    # Selector keys that may be copied onto an Article (PROVIDED_KEYS + mapped aliases).
+    SELECTABLE_SELECTOR_KEYS = (Html2rss::Article::PROVIDED_KEYS + SELECTOR_TO_ARTICLE_KEY.keys).to_set.freeze
 
     ##
     # Initializes a new Selectors instance.
@@ -45,7 +50,7 @@ module Html2rss
       @time_zone = time_zone
 
       prepare_selectors!
-      @rss_item_attributes = @selectors.keys & Html2rss::Article::PROVIDED_KEYS
+      @rss_item_attributes = @selectors.keys.select { |key| SELECTABLE_SELECTOR_KEYS.include?(key) }
     end
 
     ##
@@ -92,7 +97,13 @@ module Html2rss
     # @return [Hash] Hash of attributes for the article.
     def extract_article(item, page_response = response)
       scope = item_scope_for(item, page_response.url)
-      @rss_item_attributes.to_h { |key| [key, scope.select(key)] }.compact
+      @rss_item_attributes.each_with_object({}) do |selector_key, hash|
+        value = scope.select(selector_key)
+        next if value.nil?
+
+        article_key = SELECTOR_TO_ARTICLE_KEY.fetch(selector_key, selector_key)
+        hash[article_key] = article_key == :enclosures ? wrap_enclosure_value(value) : value
+      end
     end
 
     ##
@@ -308,9 +319,20 @@ module Html2rss
       @channel_contexts[base_url] ||= { url: base_url, time_zone: @time_zone }.freeze
     end
 
-    # @return [Hash] enclosure details.
+    # Keep a single enclosure Hash as one list entry; +Array(hash)+ would split pairs.
+    #
+    # @param value [Hash, Array] enclosure hash or list of hashes
+    # @return [Array]
+    def wrap_enclosure_value(value)
+      value.is_a?(Array) ? value : [value]
+    end
+
+    # @return [Hash, nil] enclosure details, or nil when the selector yields nothing.
     def enclosure(scope:, config:)
-      url = Url.from_relative(select_regular(:enclosure, scope:, config:), scope.base_url)
+      selected = select_regular(:enclosure, scope:, config:)
+      return if selected.nil? || selected.to_s.strip.empty?
+
+      url = Url.from_relative(selected, scope.base_url)
 
       { url:, type: config[:content_type] }
     end

data/lib/html2rss/status.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Shared RSS +generator+ / JSON Feed +user_comment+ formatter string.
+  #
+  # Exposed publicly via {FeedResult#status}. Safe to log without reading articles.
+  # Stable telemetry payload for cross-repo consumers (e.g. html2rss-web observability).
+  # Tallies and counters are validated and frozen at construction (including Marshal load).
+  Status = Data.define(
+    :version, :scraper_tallies, :dedup_dropped, :selected_strategy, :attempt_count, :strategy_attempts
+  ) do
+    class << self
+      ##
+      # Builds status from extracted articles and scrape telemetry.
+      #
+      # @param articles [Array<Html2rss::Article>] articles kept after deduplication
+      # @param dedup_dropped [Integer] number of articles removed by deduplication
+      # @param selected_strategy [Symbol, nil] concrete strategy that succeeded under +:auto+ (else +nil+)
+      # @param attempt_count [Integer] auto-fallback attempt count (0 when not under +:auto+)
+      # @param strategy_attempts [Array<Hash>] auto-fallback attempt hashes (empty outside +:auto+)
+      # @return [Html2rss::Status]
+      def build(articles:, dedup_dropped: 0, selected_strategy: nil, attempt_count: 0, strategy_attempts: [])
+        tallies = articles.filter_map(&:scraper).tally.transform_keys { |klass| scraper_name(klass) }
+        new(
+          version: Html2rss::VERSION,
+          scraper_tallies: tallies,
+          dedup_dropped:,
+          selected_strategy:,
+          attempt_count:,
+          strategy_attempts:
+        )
+      end
+
+      ##
+      # @param klass [Class, #to_s] scraper class
+      # @return [String] short scraper label for tallies / generator text
+      def scraper_name(klass)
+        klass.to_s.gsub(/(?:Html2rss|Scraper)::/, '')
+      end
+    end
+
+    ##
+    # @param version [String]
+    # @param scraper_tallies [Hash{String => Integer}]
+    # @param dedup_dropped [Integer]
+    # @param selected_strategy [Symbol, nil]
+    # @param attempt_count [Integer]
+    # @param strategy_attempts [Array<Hash>]
+    # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength -- Status Data.define members
+    def initialize(
+      version:, scraper_tallies:, dedup_dropped:, selected_strategy: nil, attempt_count: 0, strategy_attempts: []
+    )
+      dedup = Integer(dedup_dropped)
+      attempts = Integer(attempt_count)
+      validate_counters!(dedup:, attempts:, selected_strategy:)
+
+      super(
+        version: version.to_s.dup.freeze,
+        scraper_tallies: freeze_tallies(scraper_tallies),
+        dedup_dropped: dedup,
+        selected_strategy:,
+        attempt_count: attempts,
+        strategy_attempts: freeze_attempts(strategy_attempts)
+      )
+    end
+    # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
+
+    ##
+    # Observability hash for web (+scraper_status+). Omits empty/absent optional keys:
+    # +:scraper_tallies+ when empty, +:selected_strategy+ when +nil+, +:attempt_count+ when zero,
+    # +:strategy_attempts+ when empty.
+    # Data members remain available via readers even when omitted here.
+    #
+    # @return [Hash{Symbol => Object}] always +:version+ (String), +:dedup_dropped+ (Integer);
+    #   optionally +:scraper_tallies+, +:selected_strategy+, +:attempt_count+, +:strategy_attempts+
+    def to_h
+      {
+        version:,
+        dedup_dropped:,
+        **(scraper_tallies.any? ? { scraper_tallies: } : {}),
+        **(selected_strategy.nil? ? {} : { selected_strategy: }),
+        **(attempt_count.positive? ? { attempt_count: } : {}),
+        **(strategy_attempts.any? ? { strategy_attempts: } : {})
+      }
+    end
+
+    ##
+    # Formats the RSS +generator+ string and JSON Feed +user_comment+.
+    # Scraper-focused only — auto strategy summary stays on {#to_h}, not this string.
+    # Omits the +(scrapers: …)+ clause when tallies are empty.
+    #
+    # @return [String]
+    def to_generator_comment
+      return "html2rss V. #{version}" if scraper_tallies.empty?
+
+      counts = scraper_tallies.map { |name, count| "#{name} (#{count})" }
+      "html2rss V. #{version} (scrapers: #{counts.join(', ')})"
+    end
+
+    private
+
+    def marshal_dump
+      [version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts]
+    end
+
+    def marshal_load((version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts))
+      initialize(version:, scraper_tallies:, dedup_dropped:, selected_strategy:, attempt_count:, strategy_attempts:)
+    end
+
+    def freeze_tallies(tallies)
+      tallies.to_h.transform_keys(&:to_s).transform_values { |count| Integer(count) }.freeze
+    end
+
+    def freeze_attempts(attempts)
+      Array(attempts).map { |attempt| attempt.to_h.freeze }.freeze
+    end
+
+    def validate_counters!(dedup:, attempts:, selected_strategy:)
+      raise ArgumentError, 'dedup_dropped must be >= 0' if dedup.negative?
+      raise ArgumentError, 'attempt_count must be >= 0' if attempts.negative?
+
+      unless selected_strategy.nil? || selected_strategy.is_a?(Symbol)
+        raise ArgumentError, 'selected_strategy must be a Symbol or nil'
+      end
+
+      return unless selected_strategy && attempts < 1
+
+      raise ArgumentError, 'attempt_count must be >= 1 when selected_strategy is set'
+    end
+  end
+end

data/lib/html2rss/version.rb

@@ -2,6 +2,6 @@
 
 module Html2rss
   # Current application version.
-  VERSION = '0.23.0'
+  VERSION = '0.24.0'
   public_constant :VERSION
 end

data/lib/html2rss.rb

@@ -34,22 +34,34 @@ module Html2rss
     Config.load_yaml(file, feed_name)
   end
 
+  ##
+  # Returns an opaque, Marshal-cacheable result of one scrape.
+  #
+  # Prefer this when the same scrape must render as RSS and JSON Feed (e.g. web cache).
+  #
+  # @param raw_config [Hash{Symbol => Object}] feed configuration
+  # @return [Html2rss::FeedResult]
+  def self.feed_result(raw_config)
+    FeedPipeline.new(raw_config).to_result
+  end
+
   ##
   # Returns an RSS object generated from the provided configuration.
   #
   # @param raw_config [Hash{Symbol => Object}] feed configuration
   # @return [RSS::Rss] generated RSS feed
   def self.feed(raw_config)
-    FeedPipeline.new(raw_config).to_rss
+    feed_result(raw_config).to_rss
   end
 
   ##
   # Returns a JSONFeed 1.1 hash generated from the provided configuration.
   #
   # @param raw_config [Hash{Symbol => Object}] feed configuration
+  # @param feed_url [String, nil] optional self URL for the feed (JSON Feed +feed_url+)
   # @return [Hash] JSONFeed-compliant hash
-  def self.json_feed(raw_config)
-    FeedPipeline.new(raw_config).to_json_feed
+  def self.json_feed(raw_config, feed_url: nil)
+    feed_result(raw_config).to_json_feed(feed_url:)
   end
 
   # rubocop:disable Metrics/ParameterLists

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.23.0
+  version: 0.24.0
 platform: ruby
 authors:
 - Gil Desmarais
@@ -303,7 +303,6 @@ files:
 - lib/html2rss/auto_source/scraper/schema.rb
 - lib/html2rss/auto_source/scraper/schema/category_extractor.rb
 - lib/html2rss/auto_source/scraper/schema/item_list.rb
-- lib/html2rss/auto_source/scraper/schema/list_item.rb
 - lib/html2rss/auto_source/scraper/schema/thing.rb
 - lib/html2rss/auto_source/scraper/semantic_html.rb
 - lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb
@@ -326,6 +325,7 @@ files:
 - lib/html2rss/defaults.rb
 - lib/html2rss/error.rb
 - lib/html2rss/feed_builder.rb
+- lib/html2rss/feed_builder/item_presentation.rb
 - lib/html2rss/feed_builder/json_feed.rb
 - lib/html2rss/feed_builder/json_feed/item.rb
 - lib/html2rss/feed_builder/rss.rb
@@ -334,6 +334,7 @@ files:
 - lib/html2rss/feed_pipeline/auto_fallback.rb
 - lib/html2rss/feed_pipeline/runtime_policy.rb
 - lib/html2rss/feed_pipeline/strategy_plan.rb
+- lib/html2rss/feed_result.rb
 - lib/html2rss/hash_util.rb
 - lib/html2rss/html.rb
 - lib/html2rss/html/article_extractor.rb
@@ -398,6 +399,7 @@ files:
 - lib/html2rss/selectors/post_processors/sanitize_html.rb
 - lib/html2rss/selectors/post_processors/substring.rb
 - lib/html2rss/selectors/post_processors/template.rb
+- lib/html2rss/status.rb
 - lib/html2rss/url.rb
 - lib/html2rss/version.rb
 - lib/tasks/config_schema.rake
@@ -407,7 +409,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.23.0
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.24.0
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:

data/lib/html2rss/auto_source/scraper/schema/list_item.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    module Scraper
-      class Schema
-        ##
-        # @see https://schema.org/ListItem
-        class ListItem < Thing
-          # @return [String, nil] stable list-item identifier
-          def id =          (id = (schema_object.dig(:item, :@id) || super).to_s).empty? ? nil : id
-          # @return [String, nil] list-item title
-          def title =       schema_object.dig(:item, :name) || super || url&.titleized
-          # @return [String, nil] list-item description
-          def description = schema_object.dig(:item, :description) || super
-
-          # @return [Html2rss::Url, nil]
-          def url
-            return @url if defined?(@url)
-
-            item_url = schema_object.dig(:item, :url)
-            @url = item_url ? Url.from_relative(item_url, base_url || item_url) : super
-          end
-        end
-      end
-    end
-  end
-end