datalog-theme 0.8.0 → 0.9.0

data/assets/js/loader.js

@@ -80,6 +80,40 @@ const FEATURE_CONFIG = [
   {
     name: "notebook",
     test: () => document.body?.dataset.featureNotebook === "true" || document.querySelector(".notebook-output")
+  },
+  {
+    name: "reading-state",
+    // A post with the reading-state panel, or a page listing the saved articles.
+    test: () => document.querySelector("[data-reading-state], [data-reading-list]")
+  },
+  {
+    name: "corrections",
+    test: () => document.querySelector("[data-correction-report]")
+  },
+  {
+    name: "contact",
+    test: () => document.querySelector("[data-contact-form]")
+  },
+  {
+    name: "comments",
+    test: () => document.querySelector("[data-comments-thread]")
+  },
+  {
+    name: "reactions",
+    test: () => document.querySelector("[data-reactions]")
+  },
+  {
+    name: "webmentions",
+    test: () => document.querySelector("[data-webmentions]")
+  },
+  {
+    name: "subscriptions",
+    // A subscribe form, or the page the newsletter's emails link to.
+    test: () => document.querySelector("[data-subscribe], [data-subscription-manage]")
+  },
+  {
+    name: "moderation",
+    test: () => document.querySelector("[data-moderation-inbox]")
   }
 ];
 

data/datalog-theme.gemspec

@@ -26,7 +26,7 @@ Gem::Specification.new do |spec|
 
   # The sass-embedded and nokogiri releases the theme is tested with need Ruby
   # 3.2; the gemspec said 3.0.
-  spec.required_ruby_version = ">= 3.2"
+  spec.required_ruby_version = Datalog::Theme::RUBY_REQUIREMENT
   spec.require_paths = ["lib"]
   spec.bindir = "bin"
   spec.executables = ["datalog"]
@@ -79,7 +79,6 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "fastimage", "~> 2.2"
   spec.add_runtime_dependency "loofah", "~> 2.19"
   spec.add_runtime_dependency "nokogiri", "~> 1.15"
-  spec.add_runtime_dependency "mini_magick", ">= 4.12", "< 6"
   spec.add_runtime_dependency "logger", "~> 1.6"
   spec.add_runtime_dependency "thor", "~> 1.3"
   # googleauth, with the Google Cloud gems it brings, is needed only by the

data/lib/datalog/cli.rb

@@ -422,6 +422,14 @@ module Datalog
       end
     end
 
+    desc "critical-css", "Write the critical CSS a production build inlines, into _includes/critical-css"
+    method_option :critical, type: :array,
+                             desc: "Command that runs critical (default: node_modules/.bin/critical or npx critical@8)"
+    def critical_css
+      require_relative "critical_css"
+      CriticalCss.command(site_root, options[:critical], ->(*line) { say_status(*line) })
+    end
+
     register(New, "new", "new COMMAND", "Scaffold posts, notebooks, and portfolio projects")
 
     private
@@ -439,7 +447,7 @@ module Datalog
     end
 
     def ruby_supported?
-      Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.0.0")
+      Gem::Requirement.new(Datalog::Theme::RUBY_REQUIREMENT).satisfied_by?(Gem::Version.new(RUBY_VERSION))
     end
 
     def gemfile_path(root)

data/lib/datalog/critical_css.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require "date"
+require "fileutils"
+require "jekyll"
+require "open3"
+require "tmpdir"
+require "yaml"
+
+module Datalog
+  # Writes the critical CSS a production build inlines. head.html inlines
+  # _includes/critical-css/<target>.html for a page's layout (home, post, or
+  # default for every other layout) and then loads main.css without blocking
+  # rendering. Critical CSS depends on a site's own pages and styles, so the
+  # theme ships those files empty and each site writes its own with
+  # `datalog critical-css`.
+  class CriticalCss
+    TARGETS = %w[home post default].freeze
+    DEFAULT_DIMENSIONS = [{ "width" => 1920, "height" => 1080 }, { "width" => 375, "height" => 667 }].freeze
+    PACKAGE = "critical@8"
+
+    class Error < StandardError; end
+
+    attr_reader :root, :settings
+
+    # `datalog critical-css`. status reports a line as the command's shell
+    # does; an error ends the command with exit status 1.
+    def self.command(root, critical, status)
+      config = YAML.safe_load_file(File.join(root, "_config.yml"), permitted_classes: [Date, Time]) || {}
+      log = ->(message) { status.call(:critical, message, :blue) }
+      written = new(root, config["critical_css"], critical: critical, log: log).run
+      written.each_value { |path| status.call(:write, path, :green) }
+    rescue Error, Jekyll::Errors::FatalException, SystemCallError => e
+      status.call(:error, e.message, :red)
+      exit 1
+    end
+
+    # critical: the command that runs critical, as a list of words. By default
+    # the site's own node_modules/.bin/critical, or `npx --yes critical@8`.
+    def initialize(root, settings, critical: nil, log: ->(_message) {})
+      @root = File.expand_path(root)
+      @settings = settings || {}
+      @critical = critical
+      @log = log
+    end
+
+    # Returns the files written, by target. A target with no page to extract
+    # from, such as post on a site without posts, is left as it is.
+    def run
+      unless settings["enabled"] == true
+        raise Error, "critical_css.enabled is not true in _config.yml, so no page would inline critical CSS"
+      end
+
+      Dir.mktmpdir("datalog-critical") do |site_dir|
+        @log.call("Building the site for production into a temporary directory")
+        build(site_dir)
+        pages_for(site_dir).each_with_object({}) do |(target, page), written|
+          if page.nil?
+            @log.call("No page uses the #{target} layout; _includes/critical-css/#{target}.html is left as it is")
+            next
+          end
+
+          @log.call("Extracting critical CSS for #{target} from #{page}")
+          written[target] = write(target, extract(site_dir, page))
+        end
+      end
+    end
+
+    # A production build, with critical CSS turned off for it: pages then link
+    # main.css as usual, and CSS inlined by an earlier run cannot skew the result.
+    def build(destination)
+      previous = ENV.fetch("JEKYLL_ENV", nil)
+      ENV["JEKYLL_ENV"] = "production"
+      config = Jekyll.configuration(
+        "source" => root, "destination" => destination, "quiet" => true,
+        "critical_css" => settings.merge("enabled" => false)
+      )
+      Jekyll::Site.new(config).process
+    ensure
+      ENV["JEKYLL_ENV"] = previous
+    end
+
+    # The page each target extracts from: the one critical_css.pages names, or
+    # the first built page with that layout, nearest the site root first.
+    def pages_for(site_dir)
+      configured = settings["pages"] || {}
+      found = TARGETS.to_h { |target| [target, configured[target] && page_file(site_dir, configured[target])] }
+      missing = TARGETS.reject { |target| found[target] }
+
+      html_files(site_dir).each do |page|
+        break if missing.empty?
+
+        target = target_of(File.read(File.join(site_dir, page), encoding: "utf-8"))
+        next unless missing.delete(target)
+
+        found[target] = page
+      end
+      found
+    end
+
+    # Mirrors head.html: the home and post layouts have their own file, and
+    # every other page the theme's default layout renders uses default. Pages
+    # kept out of search engines, such as the search page, are not typical.
+    def target_of(html)
+      classes = html[/<body\b[^>]*\bclass="([^"]*)"/, 1].to_s.split
+      return unless classes.include?("site-body")
+      return "home" if classes.include?("layout-home")
+      return "post" if classes.include?("layout-post")
+
+      "default" unless html.match?(/<meta name="robots" content="noindex/)
+    end
+
+    # Only the site's stylesheet: the Google Fonts stylesheet would inline
+    # @font-face rules for the font files headless Chrome is served (TrueType),
+    # and fetching it made each page take over a minute.
+    def extract(site_dir, page)
+      arguments = [page, "--base", site_dir, "--css", File.join(site_dir, "assets", "css", "main.css"),
+                   *dimension_arguments, *penthouse_arguments]
+      output, errors, status = Open3.capture3(*critical_command, *arguments, chdir: site_dir)
+      unless status.success?
+        raise Error, "critical could not extract CSS from #{page}: #{errors.strip.lines.last(5).join.strip}"
+      end
+
+      css = output.strip
+      raise Error, "critical found no critical CSS in #{page}" if css.empty?
+
+      css
+    end
+
+    def write(target, css)
+      path = File.join(root, "_includes", "critical-css", "#{target}.html")
+      FileUtils.mkdir_p(File.dirname(path))
+      # The include is rendered as Liquid; raw keeps a "{{" in CSS from being read as a tag.
+      File.write(path, "{% raw %}\n#{css}\n{% endraw %}\n")
+      path
+    end
+
+    def critical_command
+      return @critical if @critical
+
+      local = File.join(root, "node_modules", ".bin", Gem.win_platform? ? "critical.cmd" : "critical")
+      File.file?(local) ? [local] : ["npx", "--yes", PACKAGE]
+    end
+
+    def dimension_arguments
+      dimensions = Array(settings["dimensions"]).grep(Hash)
+      dimensions = DEFAULT_DIMENSIONS if dimensions.empty?
+      dimensions.flat_map { |entry| ["--dimensions", "#{entry['width']}x#{entry['height']}"] }
+    end
+
+    def penthouse_arguments
+      options = settings["penthouse_options"]
+      return [] unless options.is_a?(Hash)
+
+      options.flat_map { |key, value| ["--penthouse-#{key}", value.to_s] }
+    end
+
+    def page_file(site_dir, url)
+      path = url.to_s.sub(%r{\A/+}, "")
+      path = File.join(path, "index.html") if path.empty? || path.end_with?("/")
+      path if File.file?(File.join(site_dir, path))
+    end
+
+    def html_files(site_dir)
+      Dir.glob("**/*.html", base: site_dir).sort_by { |page| [page.count("/"), page] }
+    end
+  end
+end

data/lib/datalog/plugins/comments.rb

@@ -19,16 +19,23 @@ module Datalog
       REQUIRED_FIELDS = {
         "giscus" => %w[repo repo_id category category_id],
         "utterances" => %w[repo],
-        "disqus" => %w[shortname]
+        "disqus" => %w[shortname],
+        "api" => []
       }.freeze
 
+      # The api provider's thread (components/comments-thread.html) is a Liquid
+      # include, so it shares the service forms' labels and states.
+      API_INCLUDE = "{% include components/comments-thread.html %}"
+
       def initialize(site, config = {})
         super
         @defaults = {
           "provider" => "giscus",
           "mapping" => "pathname",
           "theme" => "light",
-          "enabled_by_default" => false
+          "enabled_by_default" => false,
+          "replies" => true,
+          "moderation" => true
         }.merge(@config)
         @memo = {}
       end
@@ -67,9 +74,12 @@ module Datalog
         settings = @memo[key]
         return "" unless settings
 
+        embed = settings["embed"]
+        embed = render_api(context) if settings["provider"] == "api" && embed.to_s.empty?
+
         <<~HTML
           <div class="datalog-comments" aria-label="Comments">
-            #{settings['embed']}
+            #{embed}
           </div>
         HTML
       end
@@ -109,6 +119,8 @@ module Datalog
           render_utterances(settings)
         when "disqus"
           render_disqus(document, settings)
+        when "api"
+          "" # the thread is rendered by the tag, in its own context
         else
           %(<p class="datalog-comments-unsupported">Comments provider "#{settings['provider']}" is not supported.</p>)
         end
@@ -140,6 +152,12 @@ module Datalog
           async></script>)
       end
 
+      # The thread of the site's own backend (#253), rendered in the tag's own
+      # context so the include sees the page and the site.
+      def render_api(context)
+        Liquid::Template.parse(API_INCLUDE).render(context)
+      end
+
       def render_disqus(document, settings)
         shortname = settings["shortname"] || ""
         nonce = nonce_attribute(document)
@@ -171,10 +189,22 @@ module Datalog
       end
 
       def missing_required_fields(provider, settings)
+        return api_missing(settings) if provider.to_s == "api"
+
         required = REQUIRED_FIELDS.fetch(provider.to_s, [])
         required.select { |key| settings[key].to_s.strip.empty? }
       end
 
+      # The api provider needs a backend: its own endpoint, or the site's
+      # dynamic services (docs/dynamic-services.md).
+      def api_missing(settings)
+        services = site.config["dynamic_services"]
+        base = services.is_a?(Hash) ? services["base_url"].to_s.strip : ""
+        return [] unless settings["endpoint"].to_s.strip.empty? && base.empty?
+
+        ["endpoint (or dynamic_services.base_url)"]
+      end
+
       def logger
         Jekyll.logger
       end

data/lib/datalog/theme/installed_files.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "jekyll"
+require "json"
+require_relative "package"
+require_relative "repository_checkout"
+
+module Datalog
+  module Theme
+    # Stops a build that would serve a mix of theme versions. A site that uses
+    # the theme from a Git submodule or another checkout reads every theme file
+    # from it, but the script bundles are build output and have to be built
+    # again whenever the checkout moves. Any site can also hold copies of files
+    # the theme provides, left from copying the theme in by hand; a site's file
+    # takes the place of the theme's, and the copies fall behind on the next
+    # update.
+    module InstalledFiles
+      module_function
+
+      BUNDLE_SOURCES = "assets/js/dist/sources.json"
+      # Theme data a site has no reason to replace: the script manifest names
+      # the bundles this version of the theme ships, and the integrity hashes
+      # belong to the CDN files its layouts load. Translations are left out,
+      # since a site may change them.
+      DATA_FILES = %w[_data/js_manifest.json _data/cdn-integrity.yml].freeze
+
+      # Jekyll prints an error on one line, so the messages are written as
+      # sentences rather than laid out in lines.
+      def check_bundles(site)
+        return unless RepositoryCheckout.applies_to?(site)
+
+        root = site.theme.root
+        reasons = stale_bundle_reasons(root)
+        return if reasons.empty?
+
+        listed = reasons.first(5).join("; ")
+        listed += "; and #{reasons.size - 5} more" if reasons.size > 5
+        raise Jekyll::Errors::FatalException,
+              "The script bundles in #{File.join(root, 'assets', 'js', 'dist')} were not built from the theme " \
+              "files beside them (#{listed}). Pages would run scripts from another version of the theme. Build " \
+              "them again by running `npm ci && npm run build:js` in #{root}."
+      end
+
+      # scripts/build_js.mjs records a SHA-256 of each file it builds from.
+      def stale_bundle_reasons(root)
+        record = File.join(root, BUNDLE_SOURCES)
+        unless File.file?(record)
+          return ["#{BUNDLE_SOURCES} is missing: the bundles were never built, or built before it was recorded"]
+        end
+
+        JSON.parse(File.read(record)).fetch("sources").filter_map do |path, digest|
+          file = File.join(root, path)
+          if !File.file?(file)
+            "#{path} is gone"
+          elsif Digest::SHA256.file(file).hexdigest != digest
+            "#{path} has changed"
+          end
+        end
+      end
+
+      def check_site_copies(site)
+        theme = site.theme
+        return unless theme && theme.name == "datalog-theme"
+
+        copies = stale_asset_copies(site, theme.root) + stale_data_copies(site, theme.root)
+        return if copies.empty?
+
+        raise Jekyll::Errors::FatalException,
+              "Your site has its own copies of files datalog-theme provides, and they differ from the theme's: " \
+              "#{copies.join(', ')}. A site's file takes the place of the theme's, so pages would load scripts, " \
+              "or check their integrity, for another version of the theme. Delete these files: the theme " \
+              "provides them."
+      end
+
+      # The site's static files that stand in for theme files pages load. Jekyll
+      # leaves the theme's file out when the site has one at the same path.
+      def stale_asset_copies(site, root)
+        source = File.join(site.source, "")
+        site.static_files.filter_map do |file|
+          next unless file.path.start_with?(source)
+
+          relative = file.relative_path.delete_prefix("/")
+          next unless Package::SCRIPTS_PAGES_LOAD.any? { |entry| Package.inside?(relative, entry) }
+
+          theme_file = File.join(root, relative)
+          relative if File.file?(theme_file) && !FileUtils.identical?(file.path, theme_file)
+        end
+      end
+
+      def stale_data_copies(site, root)
+        reader = Jekyll::DataReader.new(site)
+        own = reader.read(site.config["data_dir"])
+        DATA_FILES.filter_map do |entry|
+          key = File.basename(entry, ".*")
+          theme_file = File.join(root, entry)
+          next unless own.key?(key) && File.file?(theme_file)
+          next if own[key] == reader.read_data_file(theme_file)
+
+          Dir.glob("#{key}.*", base: site.in_source_dir(site.config["data_dir"])).map do |name|
+            File.join(site.config["data_dir"], name)
+          end.first
+        end
+      end
+    end
+  end
+end
+
+Jekyll::Hooks.register :site, :post_read do |site|
+  Datalog::Theme::InstalledFiles.check_bundles(site)
+  Datalog::Theme::InstalledFiles.check_site_copies(site)
+end

data/lib/datalog/theme/repository_checkout.rb

@@ -54,9 +54,13 @@ module Datalog
         remove_demo_data(site)
       end
 
+      # A checkout also holds the script sources the bundles are built from,
+      # which no page loads. The site gets the theme assets the gem would give it.
       def remove_demo_assets(site)
-        demo_dirs = Package::DEMO_ASSETS.map { |entry| File.join(site.theme.root, entry, "") }
-        site.static_files.reject! { |file| demo_dirs.any? { |dir| file.path.start_with?(dir) } }
+        root = File.join(site.theme.root, "")
+        site.static_files.reject! do |file|
+          file.path.start_with?(root) && !Package.theme_file?(file.path.delete_prefix(root))
+        end
       end
 
       # site.data holds the theme's data with the site's merged over it. Each key

data/lib/datalog/theme/version.rb

@@ -2,6 +2,10 @@
 
 module Datalog
   module Theme
-    VERSION = "0.8.0" unless const_defined?(:VERSION)
+    VERSION = "0.9.0" unless const_defined?(:VERSION)
+    # The Ruby the gem installs on, for the gemspec and `datalog check`. The
+    # release workflow rewrites every quoted version assigned to a name ending
+    # in VERSION in this file, so this name must not end that way.
+    RUBY_REQUIREMENT = ">= 3.2" unless const_defined?(:RUBY_REQUIREMENT)
   end
 end

data/lib/datalog-theme.rb

@@ -36,3 +36,4 @@ end
 Datalog::Theme.load_plugins
 
 require_relative "datalog/theme/repository_checkout"
+require_relative "datalog/theme/installed_files"