html2rss 0.32.4 → 0.34.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a53d8374b6b4ce7551c9f1bffd0e14960995f7df7380554f1292e0a70317cf58
-  data.tar.gz: 90e91d4275c03c120924cca66230a7aecb3606f2f50c6224d019c138cae244e2
+  metadata.gz: 06fbd552569edb4a645f262cd55a156e93e658421a497865ddae2e8e8b62ced9
+  data.tar.gz: d23cc7119baaa02d2994f169386a2dc12881032dc3c44975051b634a2fbbaf87
 SHA512:
-  metadata.gz: d2543d4431b2790c89c19ab207d0cdbbc7416493ea1184347746bc26e21068415d68f025e0cc253082224ca5191a3984163e3e48ef3e727cb3b78c19cddd3d64
-  data.tar.gz: ac588fece229b58ab2ea4de91bb467fdbfaf199f43febfd8be6b39d817fc9b9b2867c21cb472949e886ad4dd50b436d9c5c149c0976a87b8cd67d09a6a62f9f6
+  metadata.gz: 0d92e97cf1eba6d0d05d1454340bcee34fc7a089641baeb446d9c0984d56b9e1827ccb82469bb28e592c5795b489de171d998da5bf94079600fecc5a29d6b2ee
+  data.tar.gz: 77d55816b01d3efb271bcb203fe964a7177df68a04dc87de3b243acc6be6b6c9ad5cb0d9fd4c9c2a8137767828c97a3a04da500f14d076364a9df133b85d1781

data/README.md

@@ -28,8 +28,8 @@ Cloud development: [Open in GitHub Codespaces](https://github.com/codespaces/new
 1. **Config** — loads and validates configuration (YAML/hash); schema via `html2rss schema` / `schema/html2rss-config.schema.json`
 2. **RequestService** — fetches pages (`default` (HTTPX), `botasaurus`, or `local_file`)
 3. **Selectors** — extracts content via CSS selectors with extractors/post-processors
-4. **AutoSource** — auto-detects content (Schema.org, JSON state, semantic HTML, structural patterns)
-5. **FeedBuilder** — assembles Article objects and renders feeds (RSS 2.0 / JSON Feed 1.1)
+4. **AutoSource** — auto-detects content (Schema.org, JSON state, semantic HTML, structural patterns); a card's `<audio>`/`<video>` file becomes its enclosure
+5. **FeedBuilder** — assembles Article objects and renders feeds (RSS 2.0 / JSON Feed 1.1), with podcast tags for audio/video feeds
 
 ```text
 Config -> Request -> Extraction -> Processing -> Building -> Output
@@ -65,6 +65,8 @@ html2rss scrape https://example.com/news --format jsonfeed --explain
 # Off-domain item URLs are dropped by default. --keep-different-domain opts in
 # and lets the remote source control published links to unrelated domains.
 html2rss scrape https://example.com/app-list.json --keep-different-domain
+# Items the page marks as sponsored are dropped by default. --keep-sponsored opts in.
+html2rss scrape https://example.com/news --keep-sponsored
 
 # Durable config workflow
 html2rss capture https://example.com/news --write feed.yml

data/lib/html2rss/article/deduplicator.rb

@@ -11,11 +11,14 @@ module Html2rss
     # description metadata. If none of these identifiers are available it
     # defaults to the article object's hash to preserve the original entry.
     class Deduplicator
+      # Raised when the deduplicator is built without an article list.
+      class InvalidError < Html2rss::ContractError; end
+
       ##
       # @param articles [Array<Html2rss::Article>]
-      # @raise [ArgumentError] if articles are not provided
+      # @raise [InvalidError] if articles are not provided
       def initialize(articles)
-        raise ArgumentError, 'articles must be provided' unless articles
+        raise InvalidError, 'articles must be provided' unless articles
 
         @articles = articles
       end

data/lib/html2rss/article/enclosure.rb

@@ -6,43 +6,203 @@ module Html2rss
   class Article
     ##
     # Represents an enclosure attached to an article (RSS / JSON Feed media).
+    #
+    # The one owner of media-file facts: URL, MIME type, byte length and playback duration,
+    # the media-kind predicates feed builders choose by, and the parsers that turn
+    # publisher-declared size and duration strings into those facts.
     class Enclosure
-      ##
-      # Guesses the content type based on the file extension of the URL.
-      #
-      # @param url [Html2rss::Url]
-      # @param default [String] default content type
-      # @return [String] guessed content type, or default
-      def self.guess_content_type_from_url(url, default: 'application/octet-stream')
-        return default unless url
-
-        path = url.path
-        ext = File.extname(path)
-        ext = ext[1..] if ext.start_with?('.')
-
-        content_type = MIME::Types.type_for(ext)
-        content_type.first&.to_s || 'application/octet-stream'
+      # Raised when an enclosure has no absolute http(s) URL or declares an unknown kind.
+      class InvalidError < Html2rss::ContractError; end
+
+      # A byte size: a number with an optional unit (+12345+, +850 KB+, +12.3 MB+, +1.5GiB+). The whitespace
+      # runs are possessive, so a long run of spaces never backtracks.
+      SIZE = /\A\s*+(\d+(?:\.\d+)?)\s*+([kmgt]i?b|b|bytes?)?\s*+\z/i
+      # Bytes per size unit; SI prefixes count in thousands, binary (+KiB+) in 1024s.
+      SIZE_UNITS = {
+        'b' => 1, 'byte' => 1, 'bytes' => 1,
+        'kb' => 1000, 'mb' => 1000**2, 'gb' => 1000**3, 'tb' => 1000**4,
+        'kib' => 1024, 'mib' => 1024**2, 'gib' => 1024**3, 'tib' => 1024**4
+      }.freeze
+      # An ISO 8601 duration of days and time parts (+PT1H2M3S+, +PT45M+, +P1DT2H+).
+      DURATION = /\AP(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?\z/
+      # A clock duration (+itunes:duration+, Media RSS): +HH:MM:SS+, +MM:SS+ or plain seconds.
+      CLOCK = /\A(?:(?:(\d+):)?(\d+):)?(\d+(?:\.\d+)?)\z/
+      # Types of web pages (embed players, article pages), never a media file: HTML, XHTML and the
+      # types MIME::Types guesses from server-page script extensions (+.php+, +.asp+, +.aspx+, +.jsp+,
+      # +.cgi+, +.cfm+). Other +text/*+ files (+.txt+, +.csv+, +.ics+, +.vtt+) are downloads.
+      PAGE_TYPES = %w[
+        text/html application/xhtml+xml text/x-php application/x-httpd-php text/asp text/aspdotnet text/x-jsp
+        text/x-cgi text/x-coldfusion
+      ].freeze
+      # Types of stream playlists (HLS, DASH) and Flash players: played through a page or player, never
+      # downloaded as a file.
+      STREAM_TYPES = %w[
+        application/vnd.apple.mpegurl application/x-mpegurl application/dash+xml application/x-shockwave-flash
+      ].freeze
+      # The type of a URL that declares none and has no known extension.
+      UNTYPED = 'application/octet-stream'
+      # Playable audio/video types by file extension. MIME::Types lists these extensions under
+      # several types and puts a non-playable or codec-level one first (+.mp4+ as +application/mp4+,
+      # +.ogg+ as +audio/vorbis+, +.m4v+ without +video/mp4+), so media extensions are looked up here first.
+      # HLS playlists (+.m3u8+) are no media file: MIME::Types types them +application/vnd.apple.mpegurl+.
+      MEDIA_TYPES = {
+        'mp4' => 'video/mp4', 'm4v' => 'video/mp4', 'ogv' => 'video/ogg', 'webm' => 'video/webm',
+        'm4a' => 'audio/mp4', 'mp3' => 'audio/mpeg', 'ogg' => 'audio/ogg', 'oga' => 'audio/ogg',
+        'opus' => 'audio/opus', 'aac' => 'audio/aac', 'wav' => 'audio/wav', 'flac' => 'audio/flac'
+      }.freeze
+      # The type of a file whose source proves its kind (an audio or video player, a Schema.org media object,
+      # a Media RSS +medium+) while its URL names no type (no or an unknown extension, or a server page): the
+      # most widely used format of that kind, since an RSS enclosure needs a concrete type and +audio/*+ is none.
+      KIND_TYPES = { audio: 'audio/mpeg', video: 'video/mp4', image: 'image/jpeg' }.freeze
+
+      class << self
+        ##
+        # Guesses the content type based on the file extension of the URL: a media extension
+        # ({MEDIA_TYPES}) gives its audio or video type, any other the first MIME::Types match.
+        #
+        # @param url [Html2rss::Url]
+        # @param default [String] default content type
+        # @return [String] guessed content type, or default
+        def guess_content_type_from_url(url, default: UNTYPED)
+          return default unless url
+
+          path = url.path
+          MEDIA_TYPES[File.extname(path).delete_prefix('.').downcase] ||
+            MIME::Types.type_for(path).first&.content_type || default
+        end
+
+        ##
+        # Parses a declared file size (Schema.org +contentSize+ and alike) into bytes.
+        #
+        # @param value [String, Integer, nil] e.g. +"12.3 MB"+, +"850 KB"+, +"12345"+
+        # @return [Integer, nil] positive byte count, or nil when unparseable
+        def parse_size(value)
+          return unless (match = SIZE.match(value.to_s))
+
+          bytes = (Float(match[1]) * SIZE_UNITS.fetch(match[2].to_s.downcase, 1)).round
+          bytes if bytes.positive?
+        end
+
+        ##
+        # Parses a declared playback duration into seconds: ISO 8601 (Schema.org +duration+) or a
+        # clock (+itunes:duration+, Media RSS +duration+).
+        #
+        # @param value [String, Integer, nil] e.g. +"PT1H2M3S"+, +"PT45M"+, +"1:02:03"+, +"45:00"+, +"2700"+
+        # @return [Integer, nil] positive seconds, or nil when unparseable
+        def parse_duration(value)
+          return unless (parts = duration_parts(value.to_s.strip.upcase))
+
+          days, hours, minutes, seconds = parts.map(&:to_f)
+          total = ((days * 86_400) + (hours * 3600) + (minutes * 60) + seconds).round
+          total if total.positive?
+        end
+
+        private
+
+        # @return [Array<String, nil>, nil] days, hours, minutes and seconds, or nil when no duration shape matches
+        def duration_parts(text)
+          DURATION.match(text)&.captures || CLOCK.match(text)&.captures&.unshift(nil)
+        end
       end
 
-      # @param url [Html2rss::Url] absolute enclosure URL
+      # @param url [Html2rss::Url] absolute http(s) enclosure URL ({Url#web?}); sources resolve declared references
+      #   with {Url.resolve_web} and drop the rest, so no feed format ships a +javascript:+, +data:+ or +ftp:+ file
       # @param type [String, nil] optional enclosure MIME type
-      # @param bytes_length [Integer] enclosure length in bytes
-      def initialize(url:, type: nil, bytes_length: 0)
-        raise ArgumentError, 'An Enclosure requires an absolute URL' if !url || !url.absolute?
+      # @param bytes_length [Integer, nil] enclosure length in bytes; nil when unknown
+      # @param duration [Integer, nil] playback length in seconds, kept only for an audio or video file ({#media?});
+      #   nil (or not positive) when unknown
+      # @param kind [Symbol, nil] +:audio+, +:video+ or +:image+ when its source proves the file's kind
+      #   ({KIND_TYPES}); nil when unknown
+      # @raise [InvalidError] when the URL is not an absolute http(s) URL or the kind is unknown
+      def initialize(url:, type: nil, bytes_length: nil, duration: nil, kind: nil)
+        raise InvalidError, 'An Enclosure requires an absolute http(s) URL' unless url&.web?
 
         @url = url
-        @type = type
-        @bytes_length = bytes_length
+        @bytes_length = bytes_length || 0
+        @type = resolved_type(type, kind_type(kind))
+        @mime = @type.downcase[/\A[^;\s]*/]
+        @duration = duration if duration.instance_of?(Integer) && duration.positive? && media?
       end
 
-      # @return [String] explicit MIME type or one inferred from URL extension
-      def type = @type || self.class.guess_content_type_from_url(url)
+      # @return [String] explicit MIME type, else one inferred from the URL extension; a file of a known kind whose
+      #   URL names no media type takes that kind's type ({KIND_TYPES}), also when it is declared {UNTYPED}
+      attr_reader :type
 
-      # @return [Integer] enclosure length in bytes
+      # @return [Integer] enclosure length in bytes; 0 when unknown
       attr_reader :bytes_length
 
-      # @return [Html2rss::Url] absolute enclosure URL
+      # @return [Integer, nil] playback length in seconds of an audio or video file; nil for any other file
+      attr_reader :duration
+
+      # @return [Html2rss::Url] absolute http(s) enclosure URL
       attr_reader :url
+
+      # @return [Boolean] whether the file is audio
+      def audio? = mime.start_with?('audio/')
+
+      # @return [Boolean] whether the file is video
+      def video? = mime.start_with?('video/')
+
+      # @return [Boolean] whether the file is audio or video
+      def media? = audio? || video?
+
+      # @return [Boolean] whether the file is an image
+      def image? = mime.start_with?('image/')
+
+      # @return [Boolean] whether the URL is a web page ({PAGE_TYPES}: HTML, XHTML, a server page), such as an
+      #   embed player
+      def page? = PAGE_TYPES.include?(mime)
+
+      # @return [Boolean] whether the URL path ends in a file extension (+.pdf+), unlike a watch page (+/watch/42+)
+      def file_extension? = !File.extname(url.path).empty?
+
+      # A watch or player page (+/watch/42+) declared as a file: a web page ({#page?}), or an untyped URL
+      # ({UNTYPED}: no declared type and no known file extension) with no file extension and no declared
+      # byte length. A declared length makes an extensionless URL a file (+/download/42+), since only a
+      # file has one.
+      #
+      # @return [Boolean] whether the URL is a watch or player page rather than a file
+      def player_page?
+        page? || (mime == UNTYPED && !file_extension? && bytes_length.zero?)
+      end
+
+      # A file a feed reader can download: no stream playlist or Flash player ({STREAM_TYPES}), and no watch
+      # or player page ({#player_page?}). {Article#enclosures} keeps only files, so no feed format ships the rest.
+      #
+      # @return [Boolean] whether the URL is a downloadable file
+      def file? = !(STREAM_TYPES.include?(mime) || player_page?)
+
+      # A downloadable document or archive (PDF, ZIP): a file ({#file?}) that is no media file or image.
+      #
+      # @return [Boolean] whether the URL is a downloadable document or archive
+      def document? = file? && !media? && !image?
+
+      private
+
+      # @return [String] the lowercase MIME type without parameters (+audio/ogg; codecs=opus+ is +audio/ogg+)
+      attr_reader :mime
+
+      # @param kind [Symbol, nil]
+      # @return [String, nil] the type a file of that kind takes when its URL names none ({KIND_TYPES})
+      # @raise [InvalidError] for a kind other than +:audio+, +:video+ or +:image+
+      def kind_type(kind)
+        return unless kind
+
+        KIND_TYPES.fetch(kind) { raise InvalidError, "Unknown enclosure kind: #{kind.inspect}" }
+      end
+
+      # The declared type, else the type guessed from the URL; a file of a known kind whose URL names no type
+      # (declared {UNTYPED}, or guessed untyped or a server page) takes that kind's type ({KIND_TYPES}) instead.
+      #
+      # @param declared [String, nil]
+      # @param kind_type [String, nil]
+      # @return [String]
+      def resolved_type(declared, kind_type)
+        return declared if declared && !(kind_type && declared.casecmp?(UNTYPED))
+
+        guessed = self.class.guess_content_type_from_url(url)
+        kind_type && (guessed == UNTYPED || PAGE_TYPES.include?(guessed)) ? kind_type : guessed
+      end
     end
   end
 end

data/lib/html2rss/article/title.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Article
+    ##
+    # Owns article title composition and the one title-noise vocabulary.
+    #
+    # Whitespace collapse ({.collapse}) is universal and also drops invisible characters ({INVISIBLE}).
+    # Composition ({.compose}) is opt-in per caller (see {Article#initialize}): configured selector titles
+    # and publisher feed titles stay verbatim.
+    #
+    # A composed title takes the form +Kicker: Headline+ (or just +Headline+, or the kicker alone when
+    # the headline is blank). Both parts are whitespace-collapsed and trimmed of stray separators
+    # ({SEPARATOR_MARK}). Only the kicker is noise-stripped by text: a leading clock time set off by a
+    # separator (+10:00 | Topic+), screen-reader field labels (+Section:+) and type labels with their
+    # marker (+Article type: Video+). The label and marker sets are small, closed and multilingual
+    # (en, de, fr, es, it, nl, pl, sv, ja), matching {LinkDestination::TextClassifier}.
+    #
+    # A Japanese headline may open with a bracket label (+【速報】+, +＜動画＞+): one naming a content type
+    # ({TYPE_MARKERS}) is removed, any other leading bracket text becomes the kicker ({LEADING_BRACKET}).
+    #
+    # Headline text is never noise-stripped: +20:15 Uhr: Neuer Krimi+ or +9:41 is the time+ is a real
+    # headline. Teaser noise in a headline is told apart by structure instead ({Html::TitlePieces}):
+    # date and time elements leave wherever they sit outside running text, elements that are only a
+    # clock or a label ({.headline_noise_piece?}) only before the first headline word, and kicker
+    # pieces are judged with {.noise_piece?}.
+    module Title
+      # Joins a kicker to its headline.
+      SEPARATOR = ': '
+      # Characters that render nothing inside a word: soft hyphen, zero-width space, word joiner and
+      # byte-order mark. Every title drops them, verbatim publisher titles included. Zero-width non-joiner
+      # and joiner (U+200C, U+200D) stay: they shape Persian, Urdu and Indic spelling and join emoji.
+      INVISIBLE = /[\u00AD\u200B\u2060\uFEFF]/
+      # Clock time of day (24h, or 12h with AM/PM). Description chip peeling reuses it. Case-sensitive:
+      # a spaced lowercase +am+ is the German preposition (+9:15 am Bahnhof+), so only +9:15am+ takes it.
+      CLOCK = /(?:[01]?\d|2[0-3]):[0-5]\d(?::[0-5]\d)?(?:\s*(?:AM|PM)|am|pm)?/
+      # A whole piece that is only a clock time, optionally with +Uhr+ or one timezone/offset token
+      # (IANA, GMT/UTC offset, or abbreviation). Description peels it as a chip suffix too.
+      CLOCK_PIECE = %r{
+        \A
+        #{CLOCK}
+        (?:\s+(?:[A-Za-z]+(?:/[A-Za-z_]+)+|(?:GMT|UTC)[+-]\d{1,2}(?::\d{2})?|[A-Za-z]{2,5}))?
+        \z
+      }ix
+      # A whole piece that is only a Japanese calendar date (+11月12日+), optionally with a year and a clock time
+      # (+11月12日 16:42+).
+      DATE_PIECE = /\A(?:\d{4}年)?\d{1,2}月\d{1,2}日(?:\s+#{CLOCK})?\z/
+      # Content-type markers (lowercase). Description drops them as whole leftover lines.
+      TYPE_MARKERS = Set[
+        'news', 'news article', 'press release', 'video', 'audio', 'podcast', 'gallery', 'live', 'live blog',
+        'full episode', 'bildergalerie', 'fotostrecke', 'liveticker', 'pressemitteilung', 'ganze folge',
+        'vidéo', 'diaporama', 'en direct', 'communiqué de presse', 'épisode complet',
+        'vídeo', 'galería', 'en directo', 'nota de prensa', 'episodio completo',
+        'diretta', 'in diretta', 'diretta streaming', 'fotogallery', 'galleria', 'comunicato stampa',
+        'puntata completa',
+        'liveblog', 'fotogalerij', 'persbericht', 'volledige aflevering',
+        'wideo', 'galeria', 'na żywo', 'relacja na żywo', 'komunikat prasowy', 'cały odcinek',
+        'bildspel', 'direkt', 'direktsändning', 'liverapport', 'livesändning', 'pressmeddelande', 'hela avsnittet',
+        '速報', '詳報', '独自', '動画', '写真', '中継', '生中継', 'ライブ', 'プレスリリース'
+      ].freeze
+      # The markers as one alternation, longest first so +live blog+ wins over +live+.
+      MARKER_ALTERNATION = Regexp.union(TYPE_MARKERS.sort_by { -_1.length }).source
+      # One separator between a kicker and its headline: +: | • · – — - / > » › →+, each only when
+      # followed by whitespace, so a sign (+–5 Grad+), a path (+/e/+) or a German »quote« is not one, nor
+      # a comparison before a number (+> 50 Prozent+).
+      SEPARATOR_MARK = %r{(?:[:|•·–—/»›→-]|>(?![[:space:]]*\d))(?=[[:space:]]|\z)}
+      # Screen-reader field labels (+Section:+, +Artikel:+) and type labels with their marker
+      # (+Article type: Video+). Each field label needs its colon: a bare +Article+ is a word. A letter with
+      # an accented variant is an alternation, not a class: with +/i+, Ruby on Linux does not fold an
+      # uppercase accented letter (+Ä+) against a class that also holds its ASCII base (+[äa]+).
+      LABEL = /
+        (?:section|category|topic|rubrik|ressort|kategorie|thema|rubrique|cat(?:é|e)gorie|th(?:è|e)me|secci(?:ó|o)n
+          |categor(?:í|i)a|tema|sezione|categoria|argomento|rubrica|sectie|categorie|onderwerp|rubriek
+          |sekcja|kategoria|temat|dzia(?:ł|l)|sektion|kategori|(?:ä|a)mne
+          |article|artikel|art(?:í|i)culo|articolo|artyku(?:ł|l)
+          |セクション|カテゴリー?|ジャンル|分類|記事)\s*[:：]
+        | (?:article\s+type|content\s+type|artikeltyp|beitragstyp|inhaltstyp|type\s+d['’]article
+            |type\s+de\s+contenu|tipo\s+de\s+art(?:í|i)culo|tipo\s+de\s+contenido
+            |tipo\s+di\s+articolo|tipo\s+di\s+contenuto|artikeltype|inhoudstype|soort\s+artikel
+            |typ\s+artyku(?:ł|l)u|typ\s+tre(?:ś|s)ci|rodzaj\s+tre(?:ś|s)ci|innehållstyp
+            |記事の種類|記事タイプ|コンテンツの種類)\s*[:：]
+          (?:\s*(?:#{MARKER_ALTERNATION})(?![[:alnum:]]))?
+      /ix
+      # A whole piece that is only a label: a screen-reader label element, never headline text.
+      LABEL_PIECE = /\A(?:#{LABEL})\z/
+      # Leading separators of a headline (+Kicker</span> – Headline+ leaves +– Headline+).
+      LEADING_SEPARATORS = /\A(?:[[:space:]]|#{SEPARATOR_MARK})+/
+      # A leading Japanese bracket (+【…】+ or +＜…＞+) of one to thirty characters, with headline text after it.
+      LEADING_BRACKET = /\A[【＜][[:space:]]*([^】＞]{1,30}?)[[:space:]]*[】＞][[:space:]]*(?=\S)/
+      # A bracket naming a content type (+【速報】+): noise as a kicker.
+      BRACKET_LABEL = /[【＜][[:space:]]*(?:#{MARKER_ALTERNATION})[[:space:]]*[】＞]/i
+      # Leading kicker noise, repeated: separators, separated clocks, labels.
+      KICKER_NOISE = /
+        \A(?:
+          [[:space:]] | #{SEPARATOR_MARK} | #{LABEL} | #{BRACKET_LABEL}
+          | #{CLOCK}(?:\s*Uhr)?(?=[[:space:]]*(?:#{SEPARATOR_MARK}|\z))
+        )+
+      /x
+      # Trailing stray separators (+Kicker :+, +Kicker -+, +Kicker ›+). A trailing +»+ closes a French
+      # quote, so it stays.
+      TRAILING_SEPARATORS = %r{(?:[[:space:]]|[:|•·–—]|(?<=[[:space:]])[-/>›→])+\z}
+      # The inline kicker boundary of +Kicker : Headline+: a whitespace-preceded colon after at most
+      # eight words ending in a letter or closing quote/bracket, so a score (+16 : 9+) keeps its colon.
+      SPACED_COLON = /\A((?:[^[:space:]]+[[:space:]]+){0,7}?[^[:space:]]*[\p{L}"'“”’»›)\]])[[:space:]]+:[[:space:]]*/
+      # A kicker that already ends a sentence (+Kicker?+) joins its headline with a space, not a colon.
+      TERMINAL_PUNCTUATION = /[?!…‽]\z/
+      private_constant :MARKER_ALTERNATION, :LEADING_BRACKET, :BRACKET_LABEL, :LABEL, :LABEL_PIECE,
+                       :LEADING_SEPARATORS, :KICKER_NOISE, :TRAILING_SEPARATORS, :SPACED_COLON, :TERMINAL_PUNCTUATION
+
+      class << self
+        ##
+        # @param text [String, nil] title text as extracted
+        # @return [String, nil] +text+ in NFC (a composed and a decomposed +Ä+ are one title) without
+        #   {INVISIBLE} characters, whitespace runs (newlines included) collapsed and trimmed, or nil when blank
+        def collapse(text)
+          collapsed = text.to_s.unicode_normalize
+          collapsed = collapsed.gsub(INVISIBLE, '') if collapsed.match?(INVISIBLE)
+          collapsed = collapsed.gsub(/[[:space:]]+/, ' ').strip
+          collapsed unless collapsed.empty?
+        end
+
+        ##
+        # @param headline [String, nil] headline text as extracted; kept verbatim apart from separators
+        # @param kicker [String, nil] kicker / eyebrow text, when the source exposes one
+        # @return [String, nil] +Kicker: Headline+, the headline, the kicker when the headline is blank,
+        #   or nil when both are blank
+        def compose(headline, kicker: nil)
+          head, top = peel_bracket(trim(headline, LEADING_SEPARATORS), trim(kicker, KICKER_NOISE))
+          return head if top.nil? || noise_kicker?(top)
+          return top unless head
+          return head if prefixed?(head, top)
+
+          "#{top}#{top.match?(TERMINAL_PUNCTUATION) ? ' ' : SEPARATOR}#{head}"
+        end
+
+        ##
+        # Whether a piece of text is only a clock time (+19:47+, +10:00 CET+). Description peels such chips.
+        #
+        # @param text [String, nil] visible text of one element or chip
+        # @return [Boolean]
+        def clock_piece?(text) = collapse(text).to_s.match?(CLOCK_PIECE)
+
+        ##
+        # Whether a piece of text is only a calendar date or a date with a clock time (+11月12日 16:42+).
+        # {Html::TitlePieces} drops such elements at either edge of the headline.
+        #
+        # @param text [String, nil] visible text of one headline descendant element
+        # @return [Boolean]
+        def date_piece?(text) = collapse(text).to_s.match?(DATE_PIECE)
+
+        ##
+        # Whether one element inside a headline is teaser noise on its own: only a clock time or only a
+        # label (+Section:+, +Article type: Video+). {Html::TitlePieces} drops such elements when they open
+        # the headline; the headline text itself is never noise-stripped.
+        #
+        # @param text [String, nil] visible text of one headline descendant element
+        # @return [Boolean]
+        def headline_noise_piece?(text)
+          piece = collapse(text).to_s
+          piece.match?(CLOCK_PIECE) || piece.match?(DATE_PIECE) || piece.match?(LABEL_PIECE)
+        end
+
+        ##
+        # Whether one child element of a kicker is noise on its own: blank, a clock time, a closed field
+        # label (+Thema:+, +Section:+) or a type marker. Any other piece ending in a colon is kicker text
+        # (+Conflict:+); {.compose} trims that colon. {Html::TitlePieces} drops noise kicker pieces.
+        #
+        # @param text [String, nil] visible text of one kicker child element
+        # @return [Boolean]
+        def noise_piece?(text)
+          piece = collapse(text)
+          piece.nil? || piece.match?(LABEL_PIECE) || piece.match?(CLOCK_PIECE) || TYPE_MARKERS.include?(piece.downcase)
+        end
+
+        private
+
+        def trim(text, prefix)
+          return unless (collapsed = collapse(text))
+
+          trimmed = collapsed.sub(prefix, '').sub(TRAILING_SEPARATORS, '').sub(SPACED_COLON, "\\1#{SEPARATOR}")
+          trimmed unless trimmed.empty?
+        end
+
+        # A leading bracket naming a content type leaves the headline; any other bracket text is the kicker
+        # unless the source already gave one (then the headline stays whole).
+        def peel_bracket(head, top)
+          return [head, top] unless (bracket = head&.match(LEADING_BRACKET))
+          return [bracket.post_match, top] if TYPE_MARKERS.include?(bracket[1].downcase)
+          return [head, top] if top
+
+          [bracket.post_match, bracket[1]]
+        end
+
+        def noise_kicker?(kicker) = TYPE_MARKERS.include?(kicker.downcase)
+
+        # Whether the headline already opens with the kicker as a whole word.
+        def prefixed?(head, kicker)
+          head[0, kicker.length].casecmp?(kicker) && !head[kicker.length].to_s.match?(/[[:alnum:]]/)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/article.rb

@@ -6,8 +6,12 @@ module Html2rss
   ##
   # Article is a simple data object representing an article extracted from a page.
   #
+  # Article owns the title via {Article::Title}. Every title is whitespace-collapsed; callers that extract titles
+  # from heuristic card markup opt in with +compose_title: true+ to join +kicker+ to +title+ (the headline) and strip
+  # noise. Configured selector titles and publisher-declared headlines (feeds, structured data) stay verbatim.
+  #
   # Description and enclosure wire presentation live in {FeedBuilder::ItemPresentation}.
-  class Article
+  class Article # rubocop:disable Metrics/ClassLength -- one reader per article attribute
     # Allowed article attributes accepted by the value object constructor.
     PROVIDED_KEYS = %i[id title description url image author guid published_at enclosures categories scraper].freeze
     # Separator used to build deterministic deduplication fingerprints.
@@ -19,36 +23,65 @@ module Html2rss
     NOT_SET = Object.new.freeze
 
     # @param id [String, nil] stable article identifier
-    # @param title [String, nil] article title
+    # @param title [String, nil] article headline
+    # @param kicker [String, nil] kicker / eyebrow text composed into the title (not stored); used only with
+    #   +compose_title+
     # @param description [String, nil] raw extracted description/content (not feed-rendered HTML)
     # @param url [String, Html2rss::Url, nil] canonical article URL
-    # @param image [String, Html2rss::Url, nil] image URL for description / JSON Feed +image+
+    # @param image [String, Html2rss::Url, nil] image URL for description / JSON Feed +image+; without one, the
+    #   first image file among +enclosures+ ({Enclosure#image?}) is the item image
     # @param author [String, nil] author name
     # @param guid [String, Array, nil] explicit GUID override
     # @param published_at [String, Time, DateTime, nil] publication timestamp
     # @param enclosures [Array<Hash{Symbol => Object}>, nil] enclosure attribute hashes
     # @param categories [Array<String>, nil] category labels
     # @param scraper [Class, nil] scraper class that produced the article
-    # rubocop:disable-next Metrics/ParameterLists -- eleven article keys stay explicit
-    def initialize(id: nil, title: nil, description: nil, url: nil, image: nil, author: nil, guid: nil,
-                   published_at: nil, enclosures: nil, categories: nil, scraper: nil)
-      @to_h = { id:, title:, description:, url:, image:, author:, guid:, published_at:, enclosures:, categories:,
-                scraper: }.compact.transform_values { freeze_option(_1) }.freeze
+    # @param media [Boolean, nil] whether the item is an audio/video object (not an article)
+    # @param compose_title [Boolean] opt in to kicker joining and noise stripping; default keeps the title verbatim
+    # rubocop:disable-next Metrics/ParameterLists -- thirteen article inputs stay explicit
+    def initialize(id: nil, title: nil, kicker: nil, description: nil, url: nil, image: nil, author: nil, guid: nil,
+                   published_at: nil, enclosures: nil, categories: nil, scraper: nil, media: nil, compose_title: false)
+      title = compose_title ? Title.compose(title, kicker:) : Title.collapse(title)
+      image = first_image(enclosures) unless present?(image)
+      @to_h = { id:, title:, description:, url:, image:, author:, guid:, published_at:,
+                enclosures:, categories:, scraper:, media: }.compact.transform_values { freeze_option(_1) }.freeze
 
       @url = @image = @guid = @enclosures = @categories = @published_at = NOT_SET
     end
 
+    # Sole owner of the "an item needs a URL" rule: auto-source ({#valid?}) and selector items
+    # ({Html2rss::Selectors#each}) both admit only linked articles.
+    # @return [Boolean] true when the article carries a URL
+    def linked? = !url.to_s.empty?
+
     # Checks if the article is valid based on the presence of URL, ID, and either title or description.
     # @return [Boolean] True if the article is valid, otherwise false.
     def valid?
-      !url.to_s.empty? && (!title.to_s.empty? || !description.to_s.empty?) && !id.to_s.empty?
+      linked? && (!title.to_s.empty? || !description.to_s.empty?) && !id.to_s.empty?
+    end
+
+    # @param new_url [String, Html2rss::Url] replacement article URL
+    # @return [Article] a copy of this article carrying +new_url+
+    def with_url(new_url) = self.class.new(**@to_h, url: new_url)
+
+    # Folds a later copy of the same article into this one: every field this copy carries wins
+    # (title, date, guid, description, scraper), and the other copy fills only the fields it lacks.
+    #
+    # @param other [Article] another copy of this article (same {Url#article_identity})
+    # @return [Article] self when +other+ adds nothing, else a copy carrying the filled fields
+    def merge(other)
+      filled = other.provided.reject { |key, _| present?(@to_h[key]) }
+      filled.empty? ? self : self.class.new(**@to_h, **filled)
     end
 
     # @return [String, nil] stable article identifier
     def id = blank_string_to_nil(@to_h[:id])
 
-    # @return [String, nil] article title
-    def title = blank_string_to_nil(@to_h[:title])
+    # @return [Boolean] whether the scraper marked this item as an audio/video object
+    def media? = @to_h[:media] == true
+
+    # @return [String, nil] whitespace-collapsed title; +Kicker: Headline+ without noise when composed
+    def title = @to_h[:title]
 
     # Raw extracted description — feed HTML enrichment is {FeedBuilder::ItemPresentation.description_for}.
     #
@@ -59,14 +92,17 @@ module Html2rss
     def url
       return @url unless @url == NOT_SET
 
-      @url = Url.sanitize(@to_h[:url])
+      @url = http_url(@to_h[:url]) || Url.sanitize(@to_h[:url])
     end
 
+    # The item image a feed reader and the description's +img+ load: an http(s) URL ({Url#web?}), never an
+    # +ftp:+ or +mailto:+ one.
+    #
     # @return [Url, nil]
     def image
       return @image unless @image == NOT_SET
 
-      @image = Url.sanitize(@to_h[:image])
+      @image = Url.sanitize(@to_h[:image]).then { _1 if _1&.web? }
     end
 
     # @return [String, nil]
@@ -88,12 +124,16 @@ module Html2rss
       dedup_from_url || dedup_from_id || dedup_from_guid || hash
     end
 
+    # The item's downloadable files ({Enclosure#file?}): a watch or player page, stream playlist or Flash player
+    # is never an enclosure, so neither feed format ships one.
+    #
     # @return [Array<Html2rss::Article::Enclosure>] normalized enclosure objects
     def enclosures
       return @enclosures unless @enclosures == NOT_SET
 
       @enclosures = Array(@to_h[:enclosures])
                     .map { |enclosure| Enclosure.new(**enclosure) }
+                    .select(&:file?)
                     .freeze
     end
 
@@ -124,8 +164,34 @@ module Html2rss
       @to_h[:scraper]
     end
 
+    protected
+
+    # @return [Hash{Symbol => Object}] the constructor payload, for {#merge}
+    def provided = @to_h
+
     private
 
+    # The item image of an article that declares none: its first image file, which stays an enclosure.
+    def first_image(enclosures)
+      Array(enclosures).each do |enclosure|
+        file = Enclosure.new(**enclosure)
+        return file.url if file.image?
+      end
+      nil
+    end
+
+    def present?(value)
+      case value
+      when String, Array then !value.empty?
+      else !value.nil?
+      end
+    end
+
+    # A {Url} is always normalized; http(s) ones need no re-parse. Others go through +Url.sanitize+.
+    def http_url(value)
+      value if value.is_a?(Url) && value.web?
+    end
+
     ##
     # Marshal only the constructor payload so lazy +NOT_SET+ sentinels do not break
     # {FeedResult} cache round-trips.