al_ext_posts 1.0.0 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f5ab66f82088899d5caa67f1247c948635fcc3b7b43e73706f9433352430d771
-  data.tar.gz: 5cadace8daa8f365709696ca53d164c9fa3cafbd694651c294a402662c964075
+  metadata.gz: 0f90ec27d26faf8b995829c3a53b194c03e4911226367cd8da1528cbeedeba30
+  data.tar.gz: 23ed5c58215230bcbc7ffcedee2c4371e148138e15e10396cf8812b15f716f26
 SHA512:
-  metadata.gz: 9f6756b08533749353a98c29212a4b8b5337c5fb9f0b75020428b99ff5691cb7dd902c00e8f03c670bfb93f652733112d4bbe1eff2378efc463804b862dec651
-  data.tar.gz: 33e31dcc2d7362ef5e9aac0687e8cce67d0b4209511365675c9f82f40a9341152b9a27dd53705b3a9ea28a3d8b9041a13c8d630fc68b2f240cd4993c37bea7ed
+  metadata.gz: 17caacb0ed2a7b3995f7b99085dd7712e3c2b8e7ca7e59a68db00773ed2668bccd757fa722c5ff33bf1bac6e0ca525185ef39e8fe1d74bd2fdbbd104e2596555
+  data.tar.gz: 63d2fbe672b0a3d2d95901e67fde4ef272e3851d1ea47cd0b9af5b6ff654c4b60cde850bb60212e4d265fe863670d4d08cf2fa4d90b4cdc7ca2d9edf2dd3f183

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 1.0.3 - 2026-07-27
+
+- Fixed external posts being published with an empty title when a fetch degraded (an unreachable page, a page with no `<title>`, or an RSS item with a blank one), which rendered as a blank but clickable row in the blog index and produced a stream of Jekyll ``Empty `slug` generated`` warnings. A readable title is now derived from the URL's last meaningful path segment, and a warning naming the URL is logged. Post slugs, and therefore post URLs, are unchanged.
+
+## 1.0.2 - 2026-07-27
+
+- Optimized external-post slug generation: the title check no longer allocates a stripped copy of the title, character filtering runs in a single regexp pass instead of two, and the source-name fallback is only built when it is actually needed. Slug output is unchanged.
+
+## 1.0.1 - 2026-05-24
+
+- Preserved source-level categories/tags while allowing RSS entries and explicit external posts to override them.
+- Hardened tests against local timezone differences and current Minitest releases.
+
 ## 0.1.0 - 2026-02-07
 
 - Initial gem release.

data/README.md

@@ -1,31 +1,21 @@
-# Al-Ext-Posts
+# al-ext-posts
 
-A Jekyll plugin that allows you to fetch and display external blog posts from RSS feeds and specific URLs in your Jekyll site.
+`al_ext_posts` imports and renders external posts for `al-folio` v1.x and compatible Jekyll sites.
 
 ## Installation
 
-Add this line to your Jekyll site's Gemfile:
-
 ```ruby
 gem 'al_ext_posts'
 ```
 
-And then execute:
-
-```bash
-$ bundle install
-```
-
-## Usage
-
-1. Add the plugin to your site's `_config.yml`:
-
 ```yaml
 plugins:
   - al_ext_posts
 ```
 
-2. Configure your external sources in `_config.yml`:
+## Usage
+
+Configure external sources in `_config.yml`:
 
 ```yaml
 external_sources:
@@ -43,20 +33,18 @@ external_sources:
         published_date: "2024-03-21"
 ```
 
-The plugin supports two types of sources:
-
-- RSS feeds: Provide the `rss_url` parameter
-- Direct URLs: Provide a list of `posts` with `url` and `published_date`
+Supported source types:
 
-Optional per-source defaults:
+- RSS feeds (`rss_url`)
+- Manual URL entries (`posts` with `url` + `published_date`)
 
-- `categories`: array of categories automatically applied to imported posts
-- `tags`: array of tags automatically applied to imported posts
+When a fetch degrades and an entry arrives without a usable title, the title is derived from the last meaningful segment of its URL and a warning naming that URL is logged, so the entry stays readable in the post listing instead of rendering as a blank row.
 
-## Development
+## Ecosystem context
 
-After checking out the repo, run `bundle install` to install dependencies.
+- Starter demo content/wiring lives in `al-folio`.
+- External post ingestion logic is owned here.
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub.
+Parser/source behavior changes should be contributed in this repository.

data/lib/al_ext_posts.rb

@@ -3,12 +3,20 @@ require 'httparty'
 require 'jekyll'
 require 'nokogiri'
 require 'time'
+require 'uri'
 
 module AlExtPosts
   class ExternalPostsGenerator < Jekyll::Generator
     safe true
     priority :high
 
+    # Extensions dropped from a URL segment before it is turned into a title,
+    # so `.../my-post.html` reads as "My Post" rather than "My Post Html".
+    PAGE_EXTENSIONS = %w[html htm xhtml shtml php asp aspx jsp cfm md markdown txt].freeze
+
+    # Last-resort title for a URL with neither a usable path nor a host.
+    FALLBACK_TITLE = 'External post'.freeze
+
     def generate(site)
       if site.config['external_sources'] != nil
         site.config['external_sources'].each do |src|
@@ -42,27 +50,22 @@ module AlExtPosts
           content: e.content,
           summary: e.summary,
           published: e.published
-        }, src)
+        }, metadata_for_post(src, e))
       end
     end
 
     def create_document(site, source_name, url, content, src = {})
-      # check if title is composed only of whitespace or foreign characters
-      if content[:title].gsub(/[^\w]/, '').strip.empty?
-        # use the source name and last url segment as fallback
-        slug = "#{source_name.downcase.strip.gsub(' ', '-').gsub(/[^\w-]/, '')}-#{url.split('/').last}"
-      else
-        # parse title from the post or use the source name and last url segment as fallback
-        slug = content[:title].downcase.strip.gsub(' ', '-').gsub(/[^\w-]/, '')
-        slug = "#{source_name.downcase.strip.gsub(' ', '-').gsub(/[^\w-]/, '')}-#{url.split('/').last}" if slug.empty?
-      end
+      # The slug is still derived from the raw title so existing post URLs keep
+      # their source-name fallback; only the published title is filled in.
+      slug = build_slug(source_name, url, content[:title])
+      title = resolve_title(content[:title], url)
 
       path = site.in_source_dir("_posts/#{slug}.md")
       doc = Jekyll::Document.new(
         path, { :site => site, :collection => site.collections['posts'] }
       )
       doc.data['external_source'] = source_name
-      doc.data['title'] = content[:title]
+      doc.data['title'] = title
       doc.data['feed_content'] = content[:content]
       doc.data['description'] = content[:summary]
       doc.data['date'] = content[:published]
@@ -80,12 +83,140 @@ module AlExtPosts
       site.collections['posts'].docs << doc
     end
 
+    # Build a filesystem-safe post slug from the title, falling back to the
+    # source name + last URL segment when the title is missing, blank, or made
+    # up entirely of non-word characters. Guards against a nil title (e.g. an
+    # RSS entry or fetched page with no <title>), which previously raised a
+    # NoMethodError and aborted the whole build.
+    def build_slug(source_name, url, title)
+      return fallback_slug(source_name, url) unless title.to_s.match?(/\w/)
+
+      slug = slugify(title)
+      slug.empty? ? fallback_slug(source_name, url) : slug
+    end
+
+    # Fallback slug built from the source name and the last URL segment. Only
+    # computed on the fallback path so the common (titled) case does no extra
+    # string work.
+    def fallback_slug(source_name, url)
+      "#{slugify(source_name)}-#{url.split('/').last}"
+    end
+
+    # Drop every character that is not a word character, space, or hyphen in a
+    # single pass, then translate the remaining spaces to hyphens. Equivalent to
+    # the previous `gsub(' ', '-').gsub(/[^\w-]/, '')` pair, but avoids the
+    # intermediate string and the second regexp scan.
+    def slugify(value)
+      value.to_s.downcase.strip.gsub(/[^\w -]/, '').tr(' ', '-')
+    end
+
+    # Title to publish for an ingested item. Fetches degrade in ways that leave
+    # no title at all (an unreachable host, a page without <title>, an RSS item
+    # with a blank one), and publishing that empty string produced a blank but
+    # clickable row in the blog index plus a stream of Jekyll "Empty `slug`
+    # generated" warnings. The source is listed in _config.yml by the user, so
+    # the entry is kept: derive a readable title from the URL and warn that the
+    # fetch degraded.
+    def resolve_title(title, url)
+      return title if title.to_s.match?(/[[:word:]]/)
+
+      derived = title_from_url(url)
+      Jekyll.logger.warn('ExternalPosts:', "No title found for #{url} - using #{derived.inspect} derived from the URL.")
+      derived
+    end
+
+    # Turn the last meaningful path segment of a URL into a human-readable
+    # title: "https://blog.google/technology/ai/gemini-update-2024/" becomes
+    # "Gemini Update 2024". Trailing slashes, query strings and fragments are
+    # ignored, percent-escapes are decoded, and a page extension is dropped.
+    # Segments carrying no letters on their own (ids, /2024/05/ date parts) are
+    # skipped in favour of an earlier one, and a URL left with no wordy segment
+    # at all - a bare domain, an all-numeric path - falls back to its host.
+    # Never raises and never returns an empty string.
+    def title_from_url(url)
+      segments = url_path_segments(url)
+      segments[-1] = strip_page_extension(segments[-1]) unless segments.empty?
+
+      title = humanize_segment(segments.reverse.find { |segment| segment.match?(/[[:alpha:]]/) })
+      return title unless title.empty?
+
+      host = url_host(url)
+      host.empty? ? FALLBACK_TITLE : host
+    end
+
+    # Path segments of a URL, percent-decoded and stripped of blanks. Falls
+    # back to trimming the query/fragment by hand for inputs URI cannot parse.
+    def url_path_segments(url)
+      path = begin
+        URI.parse(url.to_s).path.to_s
+      rescue URI::Error
+        url.to_s.scrub('').split('#', 2).first.to_s.split('?', 2).first.to_s
+      end
+
+      path.split('/').map { |segment| decode_url_segment(segment) }.reject { |segment| segment.strip.empty? }
+    end
+
+    # Host of a URL, without a leading "www.". Empty when there is none.
+    def url_host(url)
+      host = begin
+        URI.parse(url.to_s).host
+      rescue URI::Error
+        nil
+      end
+
+      host.to_s.sub(/\Awww\./i, '')
+    end
+
+    # Percent-decode a single URL segment, keeping the raw text whenever the
+    # escapes are malformed or decode to invalid bytes. Scrubbed either way so
+    # the callers below can match against it without raising.
+    def decode_url_segment(segment)
+      decoded = URI.decode_www_form_component(segment)
+      (decoded.valid_encoding? ? decoded : segment).scrub('')
+    rescue ArgumentError
+      segment.scrub('')
+    end
+
+    # Drop a trailing web-page extension, keeping the segment untouched when
+    # the extension is unknown (a version number, say) or is all there is.
+    def strip_page_extension(segment)
+      extension = File.extname(segment)
+      return segment unless PAGE_EXTENSIONS.include?(extension.delete_prefix('.').downcase)
+
+      stripped = segment.chomp(extension)
+      stripped.empty? ? segment : stripped
+    end
+
+    # Split a URL segment on its separators and capitalize each word, leaving
+    # words that are already cased alone: "google-gemini-io-2024" becomes
+    # "Google Gemini Io 2024".
+    def humanize_segment(segment)
+      segment.to_s.gsub(/[^[:alnum:]]+/, ' ').split.map { |word| word.sub(/\A[[:lower:]]/, &:upcase) }.join(' ')
+    end
+
     def fetch_from_urls(site, src)
       src['posts'].each do |post|
         puts "...fetching #{post['url']}"
         content = fetch_content_from_url(post['url'])
         content[:published] = parse_published_date(post['published_date'])
-        create_document(site, src['name'], post['url'], content, src)
+        create_document(site, src['name'], post['url'], content, metadata_for_post(src, post))
+      end
+    end
+
+    def metadata_for_post(src, post)
+      metadata = src.dup
+      %w[categories tags].each do |key|
+        value = metadata_value(post, key)
+        metadata[key] = value if value && !(value.respond_to?(:empty?) && value.empty?)
+      end
+      metadata
+    end
+
+    def metadata_value(post, key)
+      if post.respond_to?(:key?)
+        post[key] || post[key.to_sym]
+      elsif post.respond_to?(key)
+        post.public_send(key)
       end
     end
 
@@ -104,7 +235,7 @@ module AlExtPosts
       html = HTTParty.get(url).body
       parsed_html = Nokogiri::HTML(html)
 
-      title = parsed_html.at('head title')&.text.strip || ''
+      title = parsed_html.at('head title')&.text&.strip || ''
 
       description = parsed_html.at('head meta[name="description"]')&.attr('content')
       description ||= parsed_html.at('head meta[name="og:description"]')&.attr('content')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: al_ext_posts
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.3
 platform: ruby
 authors:
 - al-org
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-02-16 00:00:00.000000000 Z
+date: 2026-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jekyll
@@ -99,7 +99,7 @@ dependencies:
         version: '2.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '5.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -109,7 +109,7 @@ dependencies:
         version: '2.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -143,7 +143,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/al-org-dev/al-ext-posts
   source_code_uri: https://github.com/al-org-dev/al-ext-posts
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -158,8 +158,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
-signing_key: 
+rubygems_version: 3.5.22
+signing_key:
 specification_version: 4
 summary: Import external posts from RSS feeds and URLs
 test_files: []