al_ext_posts 1.0.1 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7854d59bf4f472e37e31cb9c397c45154a66c1803c785cbae6be5e0a6fe52936
-  data.tar.gz: 2552fdbeb58639b0eedd78589fca65518bde5bbf283fd483565543efbb77f8ad
+  metadata.gz: 0f90ec27d26faf8b995829c3a53b194c03e4911226367cd8da1528cbeedeba30
+  data.tar.gz: 23ed5c58215230bcbc7ffcedee2c4371e148138e15e10396cf8812b15f716f26
 SHA512:
-  metadata.gz: 49733ec7e66503e0ef3326333495cad2b246a657e3780e06d3d8211d4e962293bd80ead5a524e0c0e92ea47b0d13f231c9fdf392c1268ae486bdc06d2f67552b
-  data.tar.gz: 3dbc5c1e74595e275acf15c892cc432963e953d1656899ff47a284f4b6258eaa8920fc39d50ed560f68452a1922cfb33ebdab8f00e4425181f59569e107ae123
+  metadata.gz: 17caacb0ed2a7b3995f7b99085dd7712e3c2b8e7ca7e59a68db00773ed2668bccd757fa722c5ff33bf1bac6e0ca525185ef39e8fe1d74bd2fdbbd104e2596555
+  data.tar.gz: 63d2fbe672b0a3d2d95901e67fde4ef272e3851d1ea47cd0b9af5b6ff654c4b60cde850bb60212e4d265fe863670d4d08cf2fa4d90b4cdc7ca2d9edf2dd3f183

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 1.0.3 - 2026-07-27
+
+- Fixed external posts being published with an empty title when a fetch degraded (an unreachable page, a page with no `<title>`, or an RSS item with a blank one), which rendered as a blank but clickable row in the blog index and produced a stream of Jekyll ``Empty `slug` generated`` warnings. A readable title is now derived from the URL's last meaningful path segment, and a warning naming the URL is logged. Post slugs, and therefore post URLs, are unchanged.
+
+## 1.0.2 - 2026-07-27
+
+- Optimized external-post slug generation: the title check no longer allocates a stripped copy of the title, character filtering runs in a single regexp pass instead of two, and the source-name fallback is only built when it is actually needed. Slug output is unchanged.
+
 ## 1.0.1 - 2026-05-24
 
 - Preserved source-level categories/tags while allowing RSS entries and explicit external posts to override them.

data/README.md

@@ -38,6 +38,8 @@ Supported source types:
 - RSS feeds (`rss_url`)
 - Manual URL entries (`posts` with `url` + `published_date`)
 
+When a fetch degrades and an entry arrives without a usable title, the title is derived from the last meaningful segment of its URL and a warning naming that URL is logged, so the entry stays readable in the post listing instead of rendering as a blank row.
+
 ## Ecosystem context
 
 - Starter demo content/wiring lives in `al-folio`.

data/lib/al_ext_posts.rb

@@ -3,12 +3,20 @@ require 'httparty'
 require 'jekyll'
 require 'nokogiri'
 require 'time'
+require 'uri'
 
 module AlExtPosts
   class ExternalPostsGenerator < Jekyll::Generator
     safe true
     priority :high
 
+    # Extensions dropped from a URL segment before it is turned into a title,
+    # so `.../my-post.html` reads as "My Post" rather than "My Post Html".
+    PAGE_EXTENSIONS = %w[html htm xhtml shtml php asp aspx jsp cfm md markdown txt].freeze
+
+    # Last-resort title for a URL with neither a usable path nor a host.
+    FALLBACK_TITLE = 'External post'.freeze
+
     def generate(site)
       if site.config['external_sources'] != nil
         site.config['external_sources'].each do |src|
@@ -47,22 +55,17 @@ module AlExtPosts
     end
 
     def create_document(site, source_name, url, content, src = {})
-      # check if title is composed only of whitespace or foreign characters
-      if content[:title].gsub(/[^\w]/, '').strip.empty?
-        # use the source name and last url segment as fallback
-        slug = "#{source_name.downcase.strip.gsub(' ', '-').gsub(/[^\w-]/, '')}-#{url.split('/').last}"
-      else
-        # parse title from the post or use the source name and last url segment as fallback
-        slug = content[:title].downcase.strip.gsub(' ', '-').gsub(/[^\w-]/, '')
-        slug = "#{source_name.downcase.strip.gsub(' ', '-').gsub(/[^\w-]/, '')}-#{url.split('/').last}" if slug.empty?
-      end
+      # The slug is still derived from the raw title so existing post URLs keep
+      # their source-name fallback; only the published title is filled in.
+      slug = build_slug(source_name, url, content[:title])
+      title = resolve_title(content[:title], url)
 
       path = site.in_source_dir("_posts/#{slug}.md")
       doc = Jekyll::Document.new(
         path, { :site => site, :collection => site.collections['posts'] }
       )
       doc.data['external_source'] = source_name
-      doc.data['title'] = content[:title]
+      doc.data['title'] = title
       doc.data['feed_content'] = content[:content]
       doc.data['description'] = content[:summary]
       doc.data['date'] = content[:published]
@@ -80,6 +83,117 @@ module AlExtPosts
       site.collections['posts'].docs << doc
     end
 
+    # Build a filesystem-safe post slug from the title, falling back to the
+    # source name + last URL segment when the title is missing, blank, or made
+    # up entirely of non-word characters. Guards against a nil title (e.g. an
+    # RSS entry or fetched page with no <title>), which previously raised a
+    # NoMethodError and aborted the whole build.
+    def build_slug(source_name, url, title)
+      return fallback_slug(source_name, url) unless title.to_s.match?(/\w/)
+
+      slug = slugify(title)
+      slug.empty? ? fallback_slug(source_name, url) : slug
+    end
+
+    # Fallback slug built from the source name and the last URL segment. Only
+    # computed on the fallback path so the common (titled) case does no extra
+    # string work.
+    def fallback_slug(source_name, url)
+      "#{slugify(source_name)}-#{url.split('/').last}"
+    end
+
+    # Drop every character that is not a word character, space, or hyphen in a
+    # single pass, then translate the remaining spaces to hyphens. Equivalent to
+    # the previous `gsub(' ', '-').gsub(/[^\w-]/, '')` pair, but avoids the
+    # intermediate string and the second regexp scan.
+    def slugify(value)
+      value.to_s.downcase.strip.gsub(/[^\w -]/, '').tr(' ', '-')
+    end
+
+    # Title to publish for an ingested item. Fetches degrade in ways that leave
+    # no title at all (an unreachable host, a page without <title>, an RSS item
+    # with a blank one), and publishing that empty string produced a blank but
+    # clickable row in the blog index plus a stream of Jekyll "Empty `slug`
+    # generated" warnings. The source is listed in _config.yml by the user, so
+    # the entry is kept: derive a readable title from the URL and warn that the
+    # fetch degraded.
+    def resolve_title(title, url)
+      return title if title.to_s.match?(/[[:word:]]/)
+
+      derived = title_from_url(url)
+      Jekyll.logger.warn('ExternalPosts:', "No title found for #{url} - using #{derived.inspect} derived from the URL.")
+      derived
+    end
+
+    # Turn the last meaningful path segment of a URL into a human-readable
+    # title: "https://blog.google/technology/ai/gemini-update-2024/" becomes
+    # "Gemini Update 2024". Trailing slashes, query strings and fragments are
+    # ignored, percent-escapes are decoded, and a page extension is dropped.
+    # Segments carrying no letters on their own (ids, /2024/05/ date parts) are
+    # skipped in favour of an earlier one, and a URL left with no wordy segment
+    # at all - a bare domain, an all-numeric path - falls back to its host.
+    # Never raises and never returns an empty string.
+    def title_from_url(url)
+      segments = url_path_segments(url)
+      segments[-1] = strip_page_extension(segments[-1]) unless segments.empty?
+
+      title = humanize_segment(segments.reverse.find { |segment| segment.match?(/[[:alpha:]]/) })
+      return title unless title.empty?
+
+      host = url_host(url)
+      host.empty? ? FALLBACK_TITLE : host
+    end
+
+    # Path segments of a URL, percent-decoded and stripped of blanks. Falls
+    # back to trimming the query/fragment by hand for inputs URI cannot parse.
+    def url_path_segments(url)
+      path = begin
+        URI.parse(url.to_s).path.to_s
+      rescue URI::Error
+        url.to_s.scrub('').split('#', 2).first.to_s.split('?', 2).first.to_s
+      end
+
+      path.split('/').map { |segment| decode_url_segment(segment) }.reject { |segment| segment.strip.empty? }
+    end
+
+    # Host of a URL, without a leading "www.". Empty when there is none.
+    def url_host(url)
+      host = begin
+        URI.parse(url.to_s).host
+      rescue URI::Error
+        nil
+      end
+
+      host.to_s.sub(/\Awww\./i, '')
+    end
+
+    # Percent-decode a single URL segment, keeping the raw text whenever the
+    # escapes are malformed or decode to invalid bytes. Scrubbed either way so
+    # the callers below can match against it without raising.
+    def decode_url_segment(segment)
+      decoded = URI.decode_www_form_component(segment)
+      (decoded.valid_encoding? ? decoded : segment).scrub('')
+    rescue ArgumentError
+      segment.scrub('')
+    end
+
+    # Drop a trailing web-page extension, keeping the segment untouched when
+    # the extension is unknown (a version number, say) or is all there is.
+    def strip_page_extension(segment)
+      extension = File.extname(segment)
+      return segment unless PAGE_EXTENSIONS.include?(extension.delete_prefix('.').downcase)
+
+      stripped = segment.chomp(extension)
+      stripped.empty? ? segment : stripped
+    end
+
+    # Split a URL segment on its separators and capitalize each word, leaving
+    # words that are already cased alone: "google-gemini-io-2024" becomes
+    # "Google Gemini Io 2024".
+    def humanize_segment(segment)
+      segment.to_s.gsub(/[^[:alnum:]]+/, ' ').split.map { |word| word.sub(/\A[[:lower:]]/, &:upcase) }.join(' ')
+    end
+
     def fetch_from_urls(site, src)
       src['posts'].each do |post|
         puts "...fetching #{post['url']}"
@@ -121,7 +235,7 @@ module AlExtPosts
       html = HTTParty.get(url).body
       parsed_html = Nokogiri::HTML(html)
 
-      title = parsed_html.at('head title')&.text.strip || ''
+      title = parsed_html.at('head title')&.text&.strip || ''
 
       description = parsed_html.at('head meta[name="description"]')&.attr('content')
       description ||= parsed_html.at('head meta[name="og:description"]')&.attr('content')

metadata

@@ -1,13 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: al_ext_posts
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.3
 platform: ruby
 authors:
 - al-org
+autorequire:
 bindir: bin
 cert_chain: []
-date: 1980-01-02 00:00:00.000000000 Z
+date: 2026-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jekyll
@@ -98,7 +99,7 @@ dependencies:
         version: '2.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '5.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -108,7 +109,7 @@ dependencies:
         version: '2.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -142,6 +143,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/al-org-dev/al-ext-posts
   source_code_uri: https://github.com/al-org-dev/al-ext-posts
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -156,7 +158,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 3.5.22
+signing_key:
 specification_version: 4
 summary: Import external posts from RSS feeds and URLs
 test_files: []