html2rss 0.24.0 → 0.25.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e71e2d9112602603e9cbdf23907d2ad6baabc633c22bd9305094c2ab405f9ad
-  data.tar.gz: 1239aecc1f56a8ab4f5f83990671d4655127ad481907552e89fcce30af7d25ef
+  metadata.gz: 1b6b0ddc57b5cf372cc9bc6405bb21f7dc6d00aa526e234132974ccc6ef609f7
+  data.tar.gz: 59a60ef044a99686838b5498f0712a455a6c5a832c27619db0c99ba21742f6a8
 SHA512:
-  metadata.gz: fe0e1fe5a2b81886b0bda2f9087e12c42a5b309d105bcd7c1ea2268690740ee36390ad54bafb94757d758204968617569f981f489afe05572f41be28c1e4025a
-  data.tar.gz: cfad45944b508e5fafaae484cdb625e33ee43e336a2ab74b336f398c7eaab306432cf06d524ddb81b7ec5a002e77c065d7af491a6b245d0a2f5b2df6098aeda2
+  metadata.gz: 8994aa3b90d791ea8cf75df9b576669fffe505fe0895e207745ab0e591f187ab490382d00a95d49fc1073570f886d6940f27f1c1ca80ba7c57817740f48c4bc4
+  data.tar.gz: dc62483a0ade2b582fb3839778a3a518a7f1a08e781d2105c0e93c70aaa26313e153a4e30121170b8a59e8472517e6cb8d9958b9c06fa3a862358631f469c18a

data/lib/html2rss/auto_source/scraper/html.rb

@@ -1,21 +1,11 @@
 # frozen_string_literal: true
 
-require 'nokogiri'
-
 module Html2rss
   class AutoSource
     module Scraper
       ##
-      # Scrapes article-like blocks from plain HTML by looking for repeated link
-      # structures when richer structured data is unavailable.
-      #
-      # The approach is intentionally heuristic:
-      # 1. collect repeated anchor paths
-      # 2. walk upward to a shared container shape
-      # 3. extract the best anchor found inside each container
-      #
-      # This scraper is broader and noisier than `SemanticHtml`, so it acts as a
-      # fallback for pages without stronger semantic signals.
+      # Fallback HTML list/cluster scraper via SST pipeline
+      # (Normalizer → Segmenter → Scoring::Engine → Html::SstArticleExtractor).
       class Html
         include Enumerable
 
@@ -25,53 +15,44 @@ module Html2rss
         DEFAULT_MINIMUM_SELECTOR_FREQUENCY = 2
         # Number of most frequent selectors kept for container extraction.
         DEFAULT_USE_TOP_SELECTORS = 5
+        # Maximum articles materialized after eligibility filtering.
+        TOP_K = Scoring::Engine::TOP_K
 
         ##
-        # @return [Symbol] config key used to enable or configure this scraper
+        # @return [Symbol]
         def self.options_key = :html
 
         ##
-        # Probes whether the document appears to contain repeated anchor
-        # structures that this fallback scraper can cluster into article-like
-        # containers.
-        #
-        # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
-        # @return [Boolean] true when the scraper can likely extract articles
+        # @param parsed_body [Nokogiri::HTML::Document]
+        # @return [Boolean]
         def self.articles?(parsed_body)
-          new(parsed_body, url: DETECTION_BASE_URL).any?
-        end
+          return false unless parsed_body
 
-        ##
-        # Simplify an XPath selector by removing the index notation.
-        # This keeps repeated anchor paths comparable across sibling blocks.
-        #
-        # @param xpath [String] original XPath
-        # @return [String] XPath without positional indexes
-        def self.simplify_xpath(xpath)
-          Discovery::ListCandidates.simplify_xpath(xpath)
+          new(parsed_body, url: DETECTION_BASE_URL).extractable?
+        rescue ArgumentError
+          false
         end
 
-        # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML document.
-        # @param url [String] The base URL.
-        # @param extractor [Class] The extractor class to handle article extraction.
-        # @param opts [Hash] Additional options.
-        # @option opts [Integer] :minimum_selector_frequency minimum count before a selector is considered stable
-        # @option opts [Integer] :use_top_selectors number of top selectors to keep
-        def initialize(parsed_body, url:, extractor: Html2rss::Html::ArticleExtractor, **opts)
+        # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML (when +document:+ omitted)
+        # @param url [String, Html2rss::Url]
+        # @param document [SST::Document, nil] memoized SST document from AutoSource
+        # @param link_resolver [Scoring::LinkResolver, nil] shared page-scoped resolver
+        # @param opts [Hash]
+        # @option opts [Boolean] :fallback_anchorless keep anchorless cluster cards
+        # @option opts [Integer] :minimum_selector_frequency list frequency floor
+        # @option opts [Integer] :use_top_selectors list selector budget
+        def initialize(parsed_body = nil, url:, document: nil, link_resolver: nil, **opts)
           @parsed_body = parsed_body
+          @provided_document = document
+          @provided_link_resolver = link_resolver
           @url = url
-          @extractor = extractor
           @opts = opts
           @fallback_anchorless = opts.fetch(:fallback_anchorless, false)
-          @link_heuristics = LinkHeuristics.new(url)
-          @ignored_cache = {}.compare_by_identity
         end
 
-        attr_reader :parsed_body
-
         ##
-        # @yieldparam [Hash] The scraped article hash
-        # @return [Enumerator] Enumerator for the scraped articles
+        # @yieldparam article [Html2rss::Article]
+        # @return [Enumerator]
         def each
           return enum_for(:each) unless block_given?
 
@@ -79,135 +60,77 @@ module Html2rss
         end
 
         ##
-        # @return [Boolean] true when the scraper can likely extract articles
+        # @return [Boolean]
         def extractable?
-          articles.any?
-        end
-
-        ##
-        # Decides whether a traversed node has reached a useful article-like
-        # boundary for the generic HTML scraper.
-        #
-        # The predicate prefers containers that add surrounding link context,
-        # which helps the scraper move from a leaf anchor toward a repeated
-        # teaser/card wrapper.
-        #
-        # @param node [Nokogiri::XML::Node] candidate boundary node
-        # @return [Boolean] true when the node is a good extraction boundary
-        def article_tag_condition?(node)
-          # Ignore tags that are below ignored DOM chrome.
-          return false if Html2rss::Html::Navigator.ignored_container_path?(node, @ignored_cache)
-          return true if %w[body html].include?(node.name)
-          return false unless (parent = node.parent)
-
-          anchor_count(parent) > anchor_count(node)
+          ranked_segments.any?
+        rescue ArgumentError
+          false
         end
 
         private
 
         def articles
           @articles ||= begin
-            extracted = each_article_tag.filter_map do |article_tag, selected_anchor|
-              extract_article(article_tag, selected_anchor:)
+            ranked = list_ranked
+            if ranked.empty? && @fallback_anchorless
+              materialize(cluster_ranked, fallback_anchorless: true)
+            else
+              materialize(ranked)
             end
-
-            extracted += find_anchorless_articles if @fallback_anchorless
-            extracted
           end
+        rescue ArgumentError
+          []
         end
 
-        def find_anchorless_articles
-          Discovery::DomClustering.call(
-            parsed_body,
-            minimum_selector_frequency:
-          ).map do |node|
-            @extractor.new(node, base_url: @url, selected_anchor: nil, fallback_anchorless: true).call
+        def ranked_segments
+          @ranked_segments ||= begin
+            ranked = list_ranked
+            ranked = cluster_ranked if @fallback_anchorless && ranked.empty?
+            ranked
           end
         end
 
-        ##
-        # @return [Integer]
-        def minimum_selector_frequency = @opts[:minimum_selector_frequency] || DEFAULT_MINIMUM_SELECTOR_FREQUENCY
-
-        ##
-        # @return [Boolean]
-        def use_top_selectors = @opts[:use_top_selectors] || DEFAULT_USE_TOP_SELECTORS
-
-        ##
-        # @param node [Nokogiri::XML::Node]
-        # @return [Integer]
-        def anchor_count(node)
-          (@anchor_counts ||= {}.compare_by_identity)[node] ||= node.name == 'a' ? 1 : node.css('a').size
-        end
-
-        ##
-        # @param node [Nokogiri::XML::Node]
-        # @return [Boolean]
-        def relevant_anchor?(node)
-          destination_facts = @link_heuristics.destination_facts(node)
-          return false unless destination_facts
-
-          !noise_anchor?(node, destination_facts:)
+        def list_ranked
+          @list_ranked ||= rank_strategy(:list, permit_unanchored: false)
         end
 
-        ##
-        # @yield [article_tag, selected_anchor]
-        # @yieldparam article_tag [Nokogiri::XML::Node]
-        # @yieldparam selected_anchor [Nokogiri::XML::Node]
-        # @return [Enumerator, nil]
-        def each_article_tag(&block)
-          return enum_for(:each_article_tag) unless block
-
-          anchor_filter = ->(node) { relevant_anchor?(node) }
-          boundary_condition = ->(node) { article_tag_condition?(node) }
-
-          list_candidates.each_article_tag(anchor_filter:, boundary_condition:, &block)
+        def cluster_ranked
+          @cluster_ranked ||= rank_strategy(:cluster, permit_unanchored: true)
         end
 
-        ##
-        # @param article_tag [Nokogiri::XML::Node]
-        # @param selected_anchor [Nokogiri::XML::Node, nil]
-        # @return [Hash, nil]
-        def extract_article(article_tag, selected_anchor: nil)
-          selected_anchor ||= preferred_anchor_for(article_tag)
-          return unless selected_anchor
-          return if noise_anchor?(
-            selected_anchor,
-            destination_facts: @link_heuristics.destination_facts(selected_anchor),
-            container: article_tag
+        def rank_strategy(strategy, permit_unanchored:)
+          segments = Segmenter.call(
+            document,
+            base_url: @url,
+            strategy:,
+            permit_unanchored:,
+            minimum_selector_frequency:,
+            use_top_selectors:,
+            link_resolver:
           )
-
-          @extractor.call(article_tag, base_url: @url, selected_anchor:)
+          engine.select_eligible(segments, limit: TOP_K)
         end
 
-        ##
-        # @param anchor [Nokogiri::XML::Node]
-        # @param destination_facts [LinkHeuristics::DestinationFacts, nil]
-        # @return [Boolean]
-        def noise_anchor?(anchor, destination_facts:, container: nil)
-          (@noise_anchors ||= {}.compare_by_identity)[anchor] ||= begin
-            text = Html2rss::Html::Navigator.extract_visible_text(anchor).to_s.strip
-            @link_heuristics.noise_anchor?(text:, destination_facts:, anchor:, container:)
+        def materialize(ranked, fallback_anchorless: false)
+          ranked.filter_map do |entry|
+            ::Html2rss::Html::SstArticleExtractor.call(entry, base_url: @url, scraper: self.class, fallback_anchorless:)
           end
         end
 
-        ##
-        # @param article_tag [Nokogiri::XML::Node]
-        # @return [Nokogiri::XML::Node, nil]
-        def preferred_anchor_for(article_tag)
-          article_tag.css(Html2rss::Html::Navigator::MAIN_ANCHOR_SELECTOR).find { relevant_anchor?(_1) } ||
-            Html2rss::Html::Navigator.main_anchor_for(article_tag)
+        def engine
+          @engine ||= Scoring::Engine.new(link_resolver:)
         end
 
-        ##
-        # @return [Discovery::ListCandidates]
-        def list_candidates
-          Discovery::ListCandidates.new(
-            parsed_body,
-            minimum_selector_frequency:,
-            use_top_selectors:
-          )
+        def link_resolver
+          @link_resolver ||= @provided_link_resolver || Scoring::LinkResolver.new(@url)
         end
+
+        def document
+          @document ||= @provided_document || SST::Normalizer.call(@parsed_body)
+        end
+
+        def minimum_selector_frequency = @opts[:minimum_selector_frequency] || DEFAULT_MINIMUM_SELECTOR_FREQUENCY
+        def use_top_selectors = @opts[:use_top_selectors] || DEFAULT_USE_TOP_SELECTORS
       end
     end
   end

data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      class JsonState
+        # Shapes raw entries into the structure required downstream.
+        module ArticleNormalizer
+          # Preferred keys when extracting title-like values from state payloads.
+          TITLE_KEYS = %i[title headline name text].freeze
+          # Preferred keys when extracting URL-like values from state payloads.
+          URL_KEYS = %i[url link href permalink slug path canonicalUrl shortUrl].freeze
+          # Preferred keys when extracting description-like values from state payloads.
+          DESCRIPTION_KEYS = %i[description summary excerpt dek subheading].freeze
+          # Preferred keys when extracting image-like values from state payloads.
+          IMAGE_KEYS = %i[image imageUrl thumbnailUrl thumbnail src featuredImage coverImage heroImage].freeze
+          # Preferred keys when extracting publication timestamps from state payloads.
+          PUBLISHED_AT_KEYS = %i[published_at publishedAt datePublished date publicationDate pubDate updatedAt
+                                 updated_at createdAt created_at].freeze
+          # Preferred keys when extracting category-like values from state payloads.
+          CATEGORY_KEYS = %i[categories tags section sections topic topics channel].freeze
+          # Preferred keys when extracting identifier-like values from state payloads.
+          ID_KEYS = %i[id guid uuid slug key].freeze
+
+          module_function
+
+          # rubocop:disable Metrics/MethodLength
+          # @param entry [Hash] raw article entry candidate
+          # @param base_url [String, Html2rss::Url] base URL for relative link resolution
+          # @return [Hash{Symbol => Object, nil}] normalized article hash for downstream extraction
+          def normalise(entry, base_url:)
+            return unless entry.is_a?(Hash)
+
+            title = string(ValueFinder.fetch(entry, TITLE_KEYS))
+            description = string(ValueFinder.fetch(entry, DESCRIPTION_KEYS))
+            article_url = resolve_link(entry, keys: URL_KEYS, base_url:,
+                                              log_key: 'JsonState: invalid URL encountered')
+            return unless article_url
+            return if title.nil? && description.nil?
+
+            {
+              title:,
+              description:,
+              url: article_url,
+              image: resolve_link(entry, keys: IMAGE_KEYS, base_url:,
+                                         log_key: 'JsonState: invalid image URL encountered'),
+              published_at: string(ValueFinder.fetch(entry, PUBLISHED_AT_KEYS)),
+              categories: categories(entry),
+              id: identifier(entry, article_url)
+            }.compact
+          end
+          # rubocop:enable Metrics/MethodLength
+
+          # @param value [Object] candidate scalar value
+          # @return [String, nil] normalized non-empty string value
+          def string(value)
+            trimmed = value.to_s.strip
+            trimmed unless trimmed.empty?
+          end
+
+          # @param entry [Hash] raw article entry candidate
+          # @param keys [Array<String>] preferred link keys
+          # @param base_url [String, Html2rss::Url] base URL for relative link resolution
+          # @param log_key [String] structured log message key
+          # @return [Html2rss::Url, nil] resolved absolute URL
+          def resolve_link(entry, keys:, base_url:, log_key:)
+            value = ValueFinder.fetch(entry, keys)
+            value = ValueFinder.fetch(value, keys) if value.is_a?(Hash)
+            string = string(value)
+            return unless string
+
+            Url.from_relative(string, base_url)
+          rescue ArgumentError
+            Log.debug(log_key, url: string)
+            nil
+          end
+
+          # rubocop:disable Metrics/MethodLength
+          # @param entry [Hash] raw article entry candidate
+          # @return [Array<String>, nil] normalized unique categories
+          def categories(entry)
+            raw = ValueFinder.fetch(entry, CATEGORY_KEYS)
+            names = case raw
+                    when Array then raw
+                    when Hash then raw.values
+                    when String then [raw]
+                    else []
+                    end
+
+            result = names.flat_map do |value|
+              case value
+              when Hash
+                string(ValueFinder.fetch(value, %i[name title label]))
+              else
+                string(value)
+              end
+            end.compact
+
+            result.uniq!
+            result unless result.empty?
+          end
+          # rubocop:enable Metrics/MethodLength
+
+          # @param entry [Hash] raw article entry candidate
+          # @param article_url [Html2rss::Url] resolved article URL
+          # @return [String] stable article identifier fallbacking to resolved URL
+          def identifier(entry, article_url)
+            value = ValueFinder.fetch(entry, ID_KEYS)
+            value = ValueFinder.fetch(value, ID_KEYS) if value.is_a?(Hash)
+            string(value) || article_url.to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      class JsonState
+        # Identifies arrays that look like collections of article hashes.
+        module CandidateDetector
+          # Preferred keys when extracting title-like values from state payloads.
+          TITLE_KEYS = %i[title headline name text].freeze
+          # Preferred keys when extracting URL-like values from state payloads.
+          URL_KEYS = %i[url link href permalink slug path canonicalUrl shortUrl].freeze
+
+          module_function
+
+          # @param document [Hash, Array, Object] candidate document node
+          # @return [Boolean] whether the node contains article-like arrays
+          def candidate_array?(document)
+            case document
+            when Array
+              return true if array_of_articles?(document)
+
+              document.any? { traversable_candidate?(_1) }
+            when Hash then document.each_value.any? { candidate_array?(_1) }
+            else false
+            end
+          end
+
+          # @param value [Hash, Array, Object] candidate nested value
+          # @return [Boolean] whether nested value should be traversed for article candidates
+          def traversable_candidate?(value)
+            case value
+            when Array, Hash then candidate_array?(value)
+            else false
+            end
+          end
+
+          # @param array [Array<Object>] candidate list of entries
+          # @return [Boolean] whether array includes hash entries with title and URL fields
+          def array_of_articles?(array)
+            array.any? do |element|
+              next unless element.is_a?(Hash)
+
+              title_from(element) && url_from(element)
+            end
+          end
+
+          # @param object [Hash] article candidate object
+          # @return [Object, nil] detected title-like value
+          def title_from(object)
+            ValueFinder.fetch(object, TITLE_KEYS)
+          end
+
+          # @param object [Hash] article candidate object
+          # @return [Object, nil] detected URL-like value
+          def url_from(object)
+            ValueFinder.fetch(object, URL_KEYS)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      class JsonState
+        # Scans DOM nodes for JSON payloads containing article data.
+        module DocumentScanner # rubocop:disable Metrics/ModuleLength
+          # Selector for JSON-only script tags.
+          JSON_SCRIPT_SELECTOR = 'script[type="application/json"]'
+          # Regex patterns for known global JavaScript state assignments.
+          GLOBAL_ASSIGNMENT_PATTERNS = [
+            /(?:window|self|globalThis)\.__NEXT_DATA__\s*=\s*/m,
+            /(?:window|self|globalThis)\.__NUXT__\s*=\s*/m,
+            /(?:window|self|globalThis)\.STATE\s*=\s*/m,
+            /(?:window|self|globalThis)\.__REDUX_STATE__\s*=\s*/m,
+            /(?:window|self|globalThis)\.__PRELOADED_STATE__\s*=\s*/m,
+            /(?:window|self|globalThis)\.__APOLLO_STATE__\s*=\s*/m,
+            /(?:window|self|globalThis)\.__remixContext\s*=\s*/m,
+            /(?:window|self|globalThis)\.__sveltekit_data\s*=\s*/m,
+            /(?:window|self|globalThis)\.GATSBY_STATE\s*=\s*/m,
+            /(?:window|self|globalThis)\.__ember_meta\s*=\s*/m,
+            /(?:window|self|globalThis)\.angular\s*=\s*/m
+          ].freeze
+
+          # Combined regex for faster matching of global assignments.
+          GLOBAL_ASSIGNMENT_REGEXP = Regexp.union(GLOBAL_ASSIGNMENT_PATTERNS).freeze
+
+          module_function
+
+          # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+          # @return [Array<Hash, Array>] parsed JSON documents discovered in scripts
+          def json_documents(parsed_body)
+            # Use identity-based cache to avoid double-parsing of the same document.
+            # WeakMap allows the Nokogiri Document (key) to be garbage collected.
+            # rubocop:disable ThreadSafety/ClassInstanceVariable
+            (@cache ||= ObjectSpace::WeakMap.new)[parsed_body] ||=
+              script_documents(parsed_body) + assignment_documents(parsed_body)
+            # rubocop:enable ThreadSafety/ClassInstanceVariable
+          end
+
+          # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+          # @return [Array<Hash, Array>] JSON documents extracted from JSON script tags
+          def script_documents(parsed_body)
+            parsed_body.css(JSON_SCRIPT_SELECTOR).filter_map { parse_json(_1.text) }
+          end
+
+          # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+          # @return [Array<Hash, Array>] JSON documents extracted from global assignments
+          def assignment_documents(parsed_body)
+            parsed_body.css('script').filter_map { parse_assignment(_1.text) }
+          end
+
+          # @param text [String] script text that may contain a global assignment
+          # @return [Hash, Array, nil] parsed assignment payload when available
+          def parse_assignment(text)
+            payload = assignment_payload(text)
+            parse_json(payload) if payload
+          end
+
+          # @param text [String] script text to inspect for known assignment patterns
+          # @return [String, nil] extracted JSON-like assignment payload
+          def assignment_payload(text)
+            trimmed = text.to_s.strip
+            return if trimmed.empty?
+            return unless trimmed.match?(GLOBAL_ASSIGNMENT_REGEXP)
+
+            payload = trimmed.sub(GLOBAL_ASSIGNMENT_REGEXP, '')
+            extract_assignment_payload(payload)
+          end
+
+          # @param text [String] text potentially containing JSON-like payloads
+          # @return [String, nil] normalized assignment payload
+          def extract_assignment_payload(text)
+            extract_json_block(text) || text
+          end
+
+          # @param text [String] text potentially containing JSON blocks
+          # @return [String, nil] extracted JSON block spanning balanced brackets
+          def extract_json_block(text)
+            start_index = text.index(/[\[{]/)
+            return unless start_index
+
+            stop_index = scan_for_json_end(text, start_index)
+            text[start_index..stop_index] if stop_index
+          end
+
+          # @param text [String] text starting with a JSON object/array opening token
+          # @param start_index [Integer] index where JSON-like content starts
+          # @return [Integer, nil] index where the balanced JSON payload ends
+          def scan_for_json_end(text, start_index) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+            stack = []
+            in_string = false
+            escape = false
+
+            i = start_index
+            len = text.length
+            while i < len
+              char = text[i]
+
+              if in_string
+                if escape
+                  escape = false
+                elsif char == '\\'
+                  escape = true
+                elsif char == '"'
+                  in_string = false
+                end
+              else
+                case char
+                when '"' then in_string = true
+                when '{' then stack << '}'
+                when '[' then stack << ']'
+                when '}', ']'
+                  expected = stack.pop
+                  return i if expected == char && stack.empty?
+                end
+              end
+              i += 1
+            end
+
+            nil
+          end
+
+          # @param payload [String, nil] JSON payload to parse
+          # @return [Hash, Array, nil] parsed payload or nil when parsing fails
+          def parse_json(payload)
+            return unless payload
+
+            JSON.parse(payload, symbolize_names: true)
+          rescue JSON::ParserError => error
+            parse_js_object(payload, error)
+          end
+
+          # @param payload [String] JavaScript object-literal payload
+          # @param _original_error [JSON::ParserError] original JSON parse error
+          # @return [Hash, Array, nil] parsed payload after JavaScript coercion
+          def parse_js_object(payload, _original_error)
+            coerced = coerce_javascript_object(payload)
+            return unless coerced
+
+            # Some sites emit JavaScript object literals (unquoted keys, trailing commas).
+            # Coerce those payloads into valid JSON so we keep the same parsing pipeline.
+            JSON.parse(coerced, symbolize_names: true)
+          rescue JSON::ParserError => error
+            Html2rss::Log.debug("JsonState: failed to parse coerced JavaScript object (#{error.message})")
+            nil
+          end
+
+          # @param payload [String] JavaScript object-literal payload
+          # @return [String] JSON-compatible payload string
+          def coerce_javascript_object(payload)
+            string = payload.dup
+
+            # KISS approach: mutate common JS literal quirks instead of a full parser.
+            strip_trailing_commas(quote_unquoted_keys(string))
+          end
+
+          # @param jsonish [String] JSON-like string with potentially unquoted keys
+          # @return [String] payload with unquoted object keys quoted
+          def quote_unquoted_keys(jsonish)
+            jsonish.gsub(/(?<prefix>\A\s*|[{,\[]\s*)(?<key>[A-Za-z_]\w*)(?<suffix>\s*:)/) do
+              captures = Regexp.last_match.named_captures(symbolize_names: true)
+              "#{captures[:prefix]}\"#{captures[:key]}\"#{captures[:suffix]}"
+            end
+          end
+
+          # @param jsonish [String] JSON-like string with potential trailing commas
+          # @return [String] payload without trailing commas before closing tokens
+          def strip_trailing_commas(jsonish)
+            jsonish.gsub(/,(\s*[\]}])/, '\1')
+          end
+        end
+      end
+    end
+  end
+end