html2rss 0.32.4 → 0.34.0

data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb

@@ -26,149 +26,149 @@ module Html2rss
           # Combined regex for faster matching of global assignments.
           GLOBAL_ASSIGNMENT_REGEXP = Regexp.union(GLOBAL_ASSIGNMENT_PATTERNS).freeze
 
-          module_function
-
-          # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
-          # @return [Array<Hash, Array>] parsed JSON documents discovered in scripts
-          def json_documents(parsed_body)
-            # Use identity-based cache to avoid double-parsing of the same document.
-            # WeakMap allows the Nokogiri Document (key) to be garbage collected.
-            # rubocop:disable-next ThreadSafety/ClassInstanceVariable
-            (@cache ||= ObjectSpace::WeakMap.new)[parsed_body] ||=
-              script_documents(parsed_body) + assignment_documents(parsed_body)
-          end
+          class << self
+            # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+            # @return [Array<Hash, Array>] parsed JSON documents discovered in scripts
+            def json_documents(parsed_body)
+              # Use identity-based cache to avoid double-parsing of the same document.
+              # WeakMap allows the Nokogiri Document (key) to be garbage collected.
+              # rubocop:disable-next ThreadSafety/ClassInstanceVariable
+              (@cache ||= ObjectSpace::WeakMap.new)[parsed_body] ||=
+                script_documents(parsed_body) + assignment_documents(parsed_body)
+            end
 
-          # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
-          # @return [Array<Hash, Array>] JSON documents extracted from JSON script tags
-          def script_documents(parsed_body)
-            ::Html2rss::Html::Probe.scripts(parsed_body, ::Html2rss::Html::Probe::APPLICATION_JSON)
-                                   .filter_map { parse_json(_1.text) }
-          end
+            # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+            # @return [Array<Hash, Array>] JSON documents extracted from JSON script tags
+            def script_documents(parsed_body)
+              ::Html2rss::Html::Probe.scripts(parsed_body, ::Html2rss::Html::Probe::APPLICATION_JSON)
+                                     .filter_map { parse_json(_1.text) }
+            end
 
-          # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
-          # @return [Array<Hash, Array>] JSON documents extracted from global assignments
-          def assignment_documents(parsed_body)
-            parsed_body.css('script').filter_map { parse_assignment(_1.text) }
-          end
+            # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+            # @return [Array<Hash, Array>] JSON documents extracted from global assignments
+            def assignment_documents(parsed_body)
+              parsed_body.css('script').filter_map { parse_assignment(_1.text) }
+            end
 
-          # @param text [String] script text that may contain a global assignment
-          # @return [Hash, Array, nil] parsed assignment payload when available
-          def parse_assignment(text)
-            payload = assignment_payload(text)
-            parse_json(payload) if payload
-          end
+            # @param text [String] script text that may contain a global assignment
+            # @return [Hash, Array, nil] parsed assignment payload when available
+            def parse_assignment(text)
+              payload = assignment_payload(text)
+              parse_json(payload) if payload
+            end
 
-          # @param text [String] script text to inspect for known assignment patterns
-          # @return [String, nil] extracted JSON-like assignment payload
-          def assignment_payload(text)
-            trimmed = text.to_s.strip
-            return if trimmed.empty?
-            return unless trimmed.match?(GLOBAL_ASSIGNMENT_REGEXP)
+            # @param text [String] script text to inspect for known assignment patterns
+            # @return [String, nil] extracted JSON-like assignment payload
+            def assignment_payload(text)
+              trimmed = text.to_s.strip
+              return if trimmed.empty?
+              return unless trimmed.match?(GLOBAL_ASSIGNMENT_REGEXP)
 
-            payload = trimmed.sub(GLOBAL_ASSIGNMENT_REGEXP, '')
-            extract_assignment_payload(payload)
-          end
+              payload = trimmed.sub(GLOBAL_ASSIGNMENT_REGEXP, '')
+              extract_assignment_payload(payload)
+            end
 
-          # @param text [String] text potentially containing JSON-like payloads
-          # @return [String, nil] normalized assignment payload
-          def extract_assignment_payload(text)
-            extract_json_block(text) || text
-          end
+            # @param text [String] text potentially containing JSON-like payloads
+            # @return [String, nil] normalized assignment payload
+            def extract_assignment_payload(text)
+              extract_json_block(text) || text
+            end
 
-          # @param text [String] text potentially containing JSON blocks
-          # @return [String, nil] extracted JSON block spanning balanced brackets
-          def extract_json_block(text)
-            start_index = text.index(/[\[{]/)
-            return unless start_index
+            # @param text [String] text potentially containing JSON blocks
+            # @return [String, nil] extracted JSON block spanning balanced brackets
+            def extract_json_block(text)
+              start_index = text.index(/[\[{]/)
+              return unless start_index
 
-            stop_index = scan_for_json_end(text, start_index)
-            text[start_index..stop_index] if stop_index
-          end
+              stop_index = scan_for_json_end(text, start_index)
+              text[start_index..stop_index] if stop_index
+            end
 
-          # @param text [String] text starting with a JSON object/array opening token
-          # @param start_index [Integer] index where JSON-like content starts
-          # @return [Integer, nil] index where the balanced JSON payload ends
-          def scan_for_json_end(text, start_index) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-            stack = []
-            in_string = false
-            escape = false
-
-            i = start_index
-            len = text.length
-            while i < len
-              char = text[i]
-
-              if in_string
-                if escape
-                  escape = false
-                elsif char == '\\'
-                  escape = true
-                elsif char == '"'
-                  in_string = false
-                end
-              else
-                case char
-                when '"' then in_string = true
-                when '{' then stack << '}'
-                when '[' then stack << ']'
-                when '}', ']'
-                  expected = stack.pop
-                  return i if expected == char && stack.empty?
+            # @param text [String] text starting with a JSON object/array opening token
+            # @param start_index [Integer] index where JSON-like content starts
+            # @return [Integer, nil] index where the balanced JSON payload ends
+            def scan_for_json_end(text, start_index) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+              stack = []
+              in_string = false
+              escape = false
+
+              i = start_index
+              len = text.length
+              while i < len
+                char = text[i]
+
+                if in_string
+                  if escape
+                    escape = false
+                  elsif char == '\\'
+                    escape = true
+                  elsif char == '"'
+                    in_string = false
+                  end
+                else
+                  case char
+                  when '"' then in_string = true
+                  when '{' then stack << '}'
+                  when '[' then stack << ']'
+                  when '}', ']'
+                    expected = stack.pop
+                    return i if expected == char && stack.empty?
+                  end
                 end
+                i += 1
               end
-              i += 1
-            end
 
-            nil
-          end
+              nil
+            end
 
-          # @param payload [String, nil] JSON payload to parse
-          # @return [Hash, Array, nil] parsed payload or nil when parsing fails
-          def parse_json(payload)
-            return unless payload
+            # @param payload [String, nil] JSON payload to parse
+            # @return [Hash, Array, nil] parsed payload or nil when parsing fails
+            def parse_json(payload)
+              return unless payload
 
-            JSON.parse(payload, symbolize_names: true)
-          rescue JSON::ParserError => error
-            parse_js_object(payload, error)
-          end
+              JSON.parse(payload, symbolize_names: true)
+            rescue JSON::ParserError => error
+              parse_js_object(payload, error)
+            end
 
-          # @param payload [String] JavaScript object-literal payload
-          # @param _original_error [JSON::ParserError] original JSON parse error
-          # @return [Hash, Array, nil] parsed payload after JavaScript coercion
-          def parse_js_object(payload, _original_error)
-            coerced = coerce_javascript_object(payload)
-            return unless coerced
-
-            # Some sites emit JavaScript object literals (unquoted keys, trailing commas).
-            # Coerce those payloads into valid JSON so we keep the same parsing pipeline.
-            JSON.parse(coerced, symbolize_names: true)
-          rescue JSON::ParserError => error
-            Html2rss::Log.debug("JsonState: failed to parse coerced JavaScript object (#{error.message})")
-            nil
-          end
+            # @param payload [String] JavaScript object-literal payload
+            # @param _original_error [JSON::ParserError] original JSON parse error
+            # @return [Hash, Array, nil] parsed payload after JavaScript coercion
+            def parse_js_object(payload, _original_error)
+              coerced = coerce_javascript_object(payload)
+              return unless coerced
+
+              # Some sites emit JavaScript object literals (unquoted keys, trailing commas).
+              # Coerce those payloads into valid JSON so we keep the same parsing pipeline.
+              JSON.parse(coerced, symbolize_names: true)
+            rescue JSON::ParserError => error
+              Html2rss::Log.debug("JsonState: failed to parse coerced JavaScript object (#{error.message})")
+              nil
+            end
 
-          # @param payload [String] JavaScript object-literal payload
-          # @return [String] JSON-compatible payload string
-          def coerce_javascript_object(payload)
-            string = payload.dup
+            # @param payload [String] JavaScript object-literal payload
+            # @return [String] JSON-compatible payload string
+            def coerce_javascript_object(payload)
+              string = payload.dup
 
-            # KISS approach: mutate common JS literal quirks instead of a full parser.
-            strip_trailing_commas(quote_unquoted_keys(string))
-          end
+              # KISS approach: mutate common JS literal quirks instead of a full parser.
+              strip_trailing_commas(quote_unquoted_keys(string))
+            end
 
-          # @param jsonish [String] JSON-like string with potentially unquoted keys
-          # @return [String] payload with unquoted object keys quoted
-          def quote_unquoted_keys(jsonish)
-            jsonish.gsub(/(?<prefix>\A\s*|[{,\[]\s*)(?<key>[A-Za-z_]\w*)(?<suffix>\s*:)/) do
-              captures = Regexp.last_match.named_captures(symbolize_names: true)
-              "#{captures[:prefix]}\"#{captures[:key]}\"#{captures[:suffix]}"
+            # @param jsonish [String] JSON-like string with potentially unquoted keys
+            # @return [String] payload with unquoted object keys quoted
+            def quote_unquoted_keys(jsonish)
+              jsonish.gsub(/(?<prefix>\A\s*|[{,\[]\s*)(?<key>[A-Za-z_]\w*)(?<suffix>\s*:)/) do
+                captures = Regexp.last_match.named_captures(symbolize_names: true)
+                "#{captures[:prefix]}\"#{captures[:key]}\"#{captures[:suffix]}"
+              end
             end
-          end
 
-          # @param jsonish [String] JSON-like string with potential trailing commas
-          # @return [String] payload without trailing commas before closing tokens
-          def strip_trailing_commas(jsonish)
-            jsonish.gsub(/,(\s*[\]}])/, '\1')
+            # @param jsonish [String] JSON-like string with potential trailing commas
+            # @return [String] payload without trailing commas before closing tokens
+            def strip_trailing_commas(jsonish)
+              jsonish.gsub(/,(\s*[\]}])/, '\1')
+            end
           end
         end
       end

data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb

@@ -6,47 +6,39 @@ module Html2rss
       class JsonState
         # Retrieves values from heterogeneous objects by probing multiple keys.
         module ValueFinder
-          module_function
-
-          # @param object [Hash, Array] candidate container traversed during key lookup
-          # @param keys [Array<Symbol>] keys to probe in order
-          # @return [Object, nil] first matching value
-          def fetch(object, keys)
-            case object
-            when Hash then fetch_from_hash(object, keys)
-            when Array then fetch_from_array(object, keys)
+          class << self
+            # @param object [Hash, Array] candidate container traversed during key lookup
+            # @param keys [Array<Symbol>] keys to probe in order
+            # @return [Object, nil] first matching value
+            def fetch(object, keys)
+              case object
+              when Hash then fetch_from_hash(object, keys)
+              when Array then fetch_from_array(object, keys)
+              end
             end
-          end
 
-          # @param hash [Hash] hash candidate traversed during key lookup
-          # @param keys [Array<Symbol>] keys to probe in order
-          # @return [Object, nil] first matching value from hash or nested metadata
-          def fetch_from_hash(hash, keys)
-            keys.each do |key|
-              return hash[key] if hash.key?(key)
-            end
+            # @param hash [Hash] hash candidate traversed during key lookup
+            # @param keys [Array<Symbol>] keys to probe in order
+            # @return [Object, nil] value of the first present key
+            def fetch_from_hash(hash, keys)
+              keys.each do |key|
+                return hash[key] if hash.key?(key)
+              end
 
-            fetch_nested(hash[:attributes], keys) ||
-              fetch_nested(hash[:data], keys)
-          end
-
-          # @param array [Array] array whose entries may contain target keys
-          # @param keys [Array<Symbol>] keys to probe in order
-          # @return [Object, nil] first matching value from array entries
-          def fetch_from_array(array, keys)
-            array.each do |entry|
-              result = fetch(entry, keys)
-              return result if result
+              nil
             end
 
-            nil
-          end
+            # @param array [Array] array whose entries may contain target keys
+            # @param keys [Array<Symbol>] keys to probe in order
+            # @return [Object, nil] first matching value from array entries
+            def fetch_from_array(array, keys)
+              array.each do |entry|
+                result = fetch(entry, keys)
+                return result if result
+              end
 
-          # @param value [Hash, Array, nil] nested value to recurse into
-          # @param keys [Array<Symbol>] keys to probe in order
-          # @return [Object, nil] matching nested value
-          def fetch_nested(value, keys)
-            fetch(value, keys) if value
+              nil
+            end
           end
         end
       end

data/lib/html2rss/auto_source/scraper/json_state.rb

@@ -21,6 +21,7 @@ module Html2rss
 
         class << self
           # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
+          # @return [Boolean] whether any embedded JSON document holds candidate articles
           def articles?(parsed_body)
             return false unless parsed_body
 
@@ -43,7 +44,7 @@ module Html2rss
           def discover_articles(document, base_url:, &block)
             case document
             when Array then handle_array(document, base_url:, &block)
-            when Hash then document.each_value { discover_articles(_1, base_url:, &block) if traversable?(_1) }
+            when Hash then document.each_value { discover_articles(_1, base_url:, &block) }
             end
           end
 
@@ -55,20 +56,14 @@ module Html2rss
                 yield(ArticleNormalizer.normalise(entry, base_url:))
               end
             else
-              array.each { discover_articles(_1, base_url:, &block) if traversable?(_1) }
+              array.each { discover_articles(_1, base_url:, &block) }
             end
           end
-
-          def traversable?(value)
-            value.is_a?(Array) || value.is_a?(Hash)
-          end
         end
 
         # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
         # @param url [String, Html2rss::Url] page URL used to resolve relative links
-        # @param _opts [Hash] scraper-specific options
-        # @option _opts [Object] :_reserved reserved for future scraper-specific options
-        def initialize(parsed_body, url:, **_opts)
+        def initialize(parsed_body, url:)
           @parsed_body = parsed_body
           @url = url
         end

data/lib/html2rss/auto_source/scraper/meta_oembed.rb

@@ -23,7 +23,15 @@ module Html2rss
           'og:image' => :image,
           'twitter:image' => :image,
           'article:published_time' => :published_at,
-          'article:author' => :author
+          'article:author' => :author,
+          'og:video' => :video,
+          'og:video:url' => :video,
+          'og:video:secure_url' => :video,
+          'og:video:type' => :video_type,
+          'og:audio' => :audio,
+          'og:audio:url' => :audio,
+          'og:audio:secure_url' => :audio,
+          'og:audio:type' => :audio_type
         }.freeze
         private_constant :META_MAP
 
@@ -52,10 +60,8 @@ module Html2rss
         # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
         # @param url [String, Html2rss::Url] base page URL
         # @param request_session [Html2rss::RequestSession, nil] shared request session for follow-up fetches
-        # @param _opts [Hash] unused scraper-specific options
-        # @option _opts [Object] :_reserved reserved for future scraper-specific options
         # @return [void]
-        def initialize(parsed_body, url:, request_session: nil, **_opts)
+        def initialize(parsed_body, url:, request_session: nil)
           @parsed_body = parsed_body
           @url = Html2rss::Url.from_absolute(url)
           @request_session = request_session
@@ -123,6 +129,8 @@ module Html2rss
 
           response = request_session.follow_up(url: oembed_url, relation: :auto_source, origin_url: url)
           parse_oembed_response(response)
+        rescue Html2rss::ContractError
+          raise
         rescue Html2rss::Error, RequestService::UnsupportedResponseContentType, JSON::ParserError => error
           Log.warn({
             event_name: 'meta_oembed.fetch_failed',
@@ -167,10 +175,24 @@ module Html2rss
             description: meta_data[:description] || oembed_data[:html],
             author: oembed_data[:author] || meta_data[:author],
             published_at: meta_data[:published_at],
-            image: oembed_data[:thumbnail] || meta_data[:image]
+            image: oembed_data[:thumbnail] || meta_data[:image],
+            enclosures: media_enclosures(meta_data)
           }.compact
         end
 
+        # @param meta_data [Hash{Symbol => Object}] extracted OpenGraph meta tags
+        # @return [Array<Hash{Symbol => Object}>, nil] og:video / og:audio enclosures (page-type embeds skipped; no
+        #   kind, since an og:video is often a player page rather than a file)
+        def media_enclosures(meta_data)
+          found = %i[video audio].filter_map do |kind|
+            type = meta_data[:"#{kind}_type"]
+            next if type && !type.start_with?("#{kind}/")
+
+            Html2rss::Html::ArticleRules::Enclosure.from_media(meta_data[kind], type, url)
+          end
+          found unless found.empty?
+        end
+
         # @param raw_url [String, nil] raw relative or absolute URL string
         # @return [Html2rss::Url, nil] resolved absolute URL or nil
         def resolve_url(raw_url)