jekyll-aeo 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2f7c4d9f203c1e2bc36c2f3c1547d4f6056ed77cd07d250a25ee186a46bd7891
+  data.tar.gz: 88f985556de2aae0ce6a2d0c57fb0be9be36c7a2b8f275910f7ff06e9f04bd3a
+SHA512:
+  metadata.gz: 0110becea058f84e0c28790655d695f95d3f4394003795a7c1dafef7ca0c56fcecef12b3e1dfdbcc7a1bd4f194cc5bf5db3b60b0623e98642123265b2e9436b1
+  data.tar.gz: ffaa6971e9bd344a1bb30775e00653dfe0fff53cf8ff4ec8adfcf32899c88df87003fa0b8e3bbd5e9e77507b8cae9b6a932f81adc880a3e6fcb2a6b4685094be

data/lib/jekyll-aeo/commands/validate.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "json"
+
+module JekyllAeo
+  module Commands
+    class Validate < Jekyll::Command
+      class << self
+        def init_with_program(prog)
+          prog.command(:"aeo:validate") do |c|
+            c.syntax "aeo:validate [options]"
+            c.description "Validate AEO output files (llms.txt, llms-full.txt, and referenced .md files)"
+
+            c.option "destination", "-d", "--destination DESTINATION",
+                     "The destination directory to validate"
+            c.option "source", "-s", "--source SOURCE", "Custom source directory"
+            c.option "config", "--config CONFIG_FILE[,CONFIG_FILE2,...]", Array,
+                     "Custom configuration file"
+
+            c.action do |_args, options|
+              JekyllAeo::Commands::Validate.process(options)
+            end
+          end
+        end
+
+        def process(options)
+          options = configuration_from_options(options)
+          dest = options["destination"]
+          baseurl = options["baseurl"].to_s.chomp("/")
+          errors, warnings = validate(dest, baseurl)
+          report(errors, warnings)
+        end
+
+        def validate(dest, baseurl = "")
+          errors = []
+          warnings = []
+          validate_llms_txt(dest, errors)
+          validate_llms_full_txt(dest, errors)
+          validate_md_references(dest, baseurl, errors)
+          validate_domain_profile(dest, errors, warnings)
+          [errors, warnings]
+        end
+
+        private
+
+        def validate_llms_txt(dest, errors)
+          path = File.join(dest, "llms.txt")
+          unless File.exist?(path)
+            errors << "llms.txt not found at #{path}"
+            return
+          end
+
+          content = File.read(path, encoding: "utf-8")
+          return if content.start_with?("# ")
+
+          errors << "llms.txt does not start with an H1 heading (# )"
+        end
+
+        def validate_llms_full_txt(dest, errors)
+          path = File.join(dest, "llms-full.txt")
+          return unless File.exist?(path)
+
+          return unless File.empty?(path)
+
+          errors << "llms-full.txt is empty"
+        end
+
+        def validate_md_references(dest, baseurl, errors)
+          llms_path = File.join(dest, "llms.txt")
+          return unless File.exist?(llms_path)
+
+          content = File.read(llms_path, encoding: "utf-8")
+          md_urls = content.scan(/\[.*?\]\(([^)]*\.md)\)/).flatten
+
+          md_urls.each do |url|
+            relative_url = if !baseurl.empty? && url.start_with?(baseurl)
+                             url.delete_prefix(baseurl)
+                           else
+                             url
+                           end
+            file_path = File.join(dest, relative_url)
+            errors << "Referenced file not found: #{url} (expected at #{file_path})" unless File.exist?(file_path)
+          end
+        end
+
+        def validate_domain_profile(dest, errors, warnings)
+          path = File.join(dest, ".well-known", "domain-profile.json")
+          return unless File.exist?(path)
+
+          begin
+            data = JSON.parse(File.read(path, encoding: "utf-8"))
+          rescue JSON::ParserError => e
+            errors << "domain-profile.json is not valid JSON: #{e.message}"
+            return
+          end
+
+          %w[spec name description website contact].each do |field|
+            value = data[field]
+            if value.nil? || (value.is_a?(String) && value.empty?)
+              errors << "domain-profile.json missing required field: #{field}"
+            end
+          end
+
+          entity_type = data["entity_type"]
+          return unless entity_type && !JekyllAeo::Generators::DomainProfile::VALID_ENTITY_TYPES.include?(entity_type)
+
+          warnings << "domain-profile.json has invalid entity_type: #{entity_type}"
+        end
+
+        def report(errors, warnings)
+          if errors.empty? && warnings.empty?
+            Jekyll.logger.info "AEO Validate:", "All checks passed!"
+            return
+          end
+
+          warnings.each { |w| Jekyll.logger.warn "AEO Warning:", w }
+          errors.each { |e| Jekyll.logger.error "AEO Error:", e }
+
+          summary_parts = []
+          summary_parts << "#{errors.size} error(s)" unless errors.empty?
+          summary_parts << "#{warnings.size} warning(s)" unless warnings.empty?
+          Jekyll.logger.info "AEO Validate:", summary_parts.join(", ")
+          abort if errors.any?
+        end
+      end
+    end
+  end
+end

data/lib/jekyll-aeo/config.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module JekyllAeo
+  module Config
+    DEFAULTS = {
+      "enabled" => true,
+      "exclude" => [],
+      "dotmd" => {
+        "link_tag" => "auto",
+        "include_last_modified" => true,
+        "dotmd_metadata" => false,
+        "md2dotmd" => {
+          "strip_block_tags" => true,
+          "protect_indented_code" => false
+        },
+        "html2dotmd" => {
+          "enabled" => false,
+          "selector" => nil
+        }
+      },
+      "llms_txt" => {
+        "enabled" => true,
+        "description" => nil,
+        "sections" => nil,
+        "front_matter_keys" => [],
+        "show_lastmod" => false,
+        "include_descriptions" => true
+      },
+      "llms_full_txt" => {
+        "enabled" => true,
+        "description" => nil,
+        "full_txt_mode" => "all"
+      },
+      "url_map" => {
+        "enabled" => false,
+        "output_filepath" => "docs/Url-Map.md",
+        "columns" => %w[page_id url lang layout path redirects markdown_copy skipped],
+        "show_created_at" => true
+      },
+      "robots_txt" => {
+        "enabled" => false,
+        "allow" => %w[Googlebot Bingbot OAI-SearchBot ChatGPT-User Claude-SearchBot
+                      Claude-User PerplexityBot Applebot-Extended],
+        "disallow" => %w[GPTBot ClaudeBot Google-Extended Meta-ExternalAgent Amazonbot],
+        "include_sitemap" => true,
+        "include_llms_txt" => true,
+        "custom_rules" => []
+      },
+      "domain_profile" => {
+        "enabled" => false,
+        "name" => nil,
+        "description" => nil,
+        "website" => nil,
+        "contact" => nil,
+        "logo" => nil,
+        "entity_type" => nil,
+        "jsonld" => nil
+      }
+    }.freeze
+
+    def self.from_site(site)
+      user_config = site.config["jekyll_aeo"] || {}
+      deep_merge(DEFAULTS, user_config)
+    end
+
+    # Strict-schema merge: iterates only over keys defined in +defaults+,
+    # intentionally dropping any user-supplied keys that are not part of
+    # the known schema.  This prevents typos from silently propagating and
+    # keeps the config surface predictable for downstream consumers.
+    def self.deep_merge(defaults, overrides)
+      defaults.each_with_object({}) do |(key, default_val), result|
+        override_val = overrides[key]
+
+        result[key] = if default_val.is_a?(Hash) && override_val.is_a?(Hash)
+                        deep_merge(default_val, override_val)
+                      elsif overrides.key?(key)
+                        override_val
+                      else
+                        default_val
+                      end
+      end
+    end
+
+    private_class_method :deep_merge
+  end
+end

data/lib/jekyll-aeo/generators/domain_profile.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+
+module JekyllAeo
+  module Generators
+    module DomainProfile
+      SPEC_URL = "https://ai-domain-data.org/spec/v0.1"
+
+      VALID_ENTITY_TYPES = %w[
+        Organization Person Blog NGO Community
+        Project CreativeWork SoftwareApplication Thing
+      ].freeze
+
+      def self.generate(site)
+        config = JekyllAeo::Config.from_site(site)
+        return if config["enabled"] == false
+
+        dp_config = config["domain_profile"] || {}
+        return if dp_config["enabled"] == false
+
+        profile = build_profile(dp_config, site)
+        return unless profile
+
+        output_path = File.join(site.dest, ".well-known", "domain-profile.json")
+        FileUtils.mkdir_p(File.dirname(output_path))
+        File.write(output_path, "#{JSON.pretty_generate(profile)}\n")
+      end
+
+      def self.build_profile(dp_config, site)
+        name = dp_config["name"] || site.config["title"] || site.config["name"]
+        description = dp_config["description"] || site.config["description"]
+        website = dp_config["website"] || site.config["url"]
+        contact = dp_config["contact"]
+
+        missing = { "contact" => contact, "name" => name, "description" => description, "website" => website }
+                  .select { |_, v| v.nil? || v.to_s.empty? }
+                  .keys
+        unless missing.empty?
+          Jekyll.logger.warn "AEO Domain Profile:",
+                             "Skipped — required field(s) missing: #{missing.join(', ')}"
+          return nil
+        end
+
+        profile = {
+          "spec" => SPEC_URL,
+          "name" => name.to_s,
+          "description" => description.to_s,
+          "website" => website.to_s,
+          "contact" => contact.to_s
+        }
+
+        add_optional_fields(profile, dp_config)
+        profile
+      end
+
+      def self.add_optional_fields(profile, dp_config)
+        entity_type = dp_config["entity_type"]
+        if entity_type
+          if VALID_ENTITY_TYPES.include?(entity_type)
+            profile["entity_type"] = entity_type
+          else
+            Jekyll.logger.warn "AEO Domain Profile:",
+                               "Invalid entity_type '#{entity_type}' — ignored. " \
+                               "Valid: #{VALID_ENTITY_TYPES.join(', ')}"
+          end
+        end
+        profile["logo"] = dp_config["logo"] if dp_config["logo"]
+        jsonld = dp_config["jsonld"]
+        profile["jsonld"] = jsonld if jsonld.is_a?(Hash)
+      end
+
+      private_class_method :build_profile, :add_optional_fields
+    end
+  end
+end

data/lib/jekyll-aeo/generators/llms_full_txt.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module JekyllAeo
+  module Generators
+    module LlmsFullTxt
+      def self.generate(site)
+        config = JekyllAeo::Config.from_site(site)
+        return if config["enabled"] == false
+
+        full_config = config["llms_full_txt"] || {}
+        return if full_config["enabled"] == false
+
+        warn_deprecated_full_txt_mode(site.config["jekyll_aeo"])
+
+        llms_config = config["llms_txt"] || {}
+        eligible = LlmsTxt.collect_eligible(site, config)
+        sections = LlmsTxt.build_sections(eligible, llms_config)
+
+        content = build_content(site, sections, eligible, full_config)
+        File.write(File.join(site.dest, "llms-full.txt"), content)
+      end
+
+      def self.build_content(site, sections, eligible, full_config)
+        mode = full_config.fetch("full_txt_mode", "all")
+
+        items_to_include = if mode == "linked"
+                             sections.flat_map { |s| s[:items] }
+                           else
+                             eligible
+                           end
+
+        lines = []
+        lines << "# #{site.config['title']}"
+        lines << ""
+
+        description = full_config["description"] || site.config["description"]
+        if description && !description.to_s.empty?
+          lines << "> #{description}"
+          lines << ""
+        end
+
+        items_to_include.each do |item|
+          next unless File.exist?(item[:dest_md])
+
+          content = File.read(item[:dest_md], encoding: "utf-8")
+          lines << "---"
+          lines << ""
+          lines << content.strip
+          lines << ""
+        end
+
+        "#{lines.join("\n").rstrip}\n"
+      end
+
+      def self.warn_deprecated_full_txt_mode(user_aeo_config)
+        user_aeo_config ||= {}
+        llms_txt_config = user_aeo_config["llms_txt"] || {}
+        return unless llms_txt_config.key?("full_txt_mode")
+
+        Jekyll.logger.warn "AEO:", "'llms_txt.full_txt_mode' has moved to 'llms_full_txt.full_txt_mode'"
+      end
+
+      private_class_method :build_content, :warn_deprecated_full_txt_mode
+    end
+  end
+end

data/lib/jekyll-aeo/generators/llms_txt.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module JekyllAeo
+  module Generators
+    module LlmsTxt
+      def self.generate(site)
+        config = JekyllAeo::Config.from_site(site)
+        return if config["enabled"] == false
+
+        llms_config = config["llms_txt"] || {}
+        return if llms_config["enabled"] == false
+
+        eligible = collect_eligible(site, config)
+        sections = build_sections(eligible, llms_config)
+
+        llms_txt = build_llms_txt(site, sections, llms_config, config)
+        File.write(File.join(site.dest, "llms.txt"), llms_txt)
+      end
+
+      def self.collect_eligible(site, config)
+        items = []
+
+        site.documents.each do |doc|
+          next if JekyllAeo::Utils::SkipLogic.skip?(doc, site, config)
+
+          items << {
+            obj: doc,
+            title: doc.data["title"] || "",
+            description: doc.data["description"] || "",
+            url: doc.url,
+            collection: doc.collection&.label,
+            dest_md: md_dest_path(doc, site)
+          }
+        end
+
+        site.pages.each do |page|
+          next if JekyllAeo::Utils::SkipLogic.skip?(page, site, config)
+
+          items << {
+            obj: page,
+            title: page.data["title"] || "",
+            description: page.data["description"] || "",
+            url: page.url,
+            collection: nil,
+            dest_md: md_dest_path(page, site)
+          }
+        end
+
+        items
+      end
+
+      def self.build_sections(eligible, llms_config)
+        custom_sections = llms_config["sections"]
+
+        if custom_sections
+          build_custom_sections(eligible, custom_sections)
+        else
+          build_auto_sections(eligible)
+        end
+      end
+
+      def self.md_dest_path(obj, site)
+        JekyllAeo::Utils::MdUrl.dest_path(obj, site)
+      end
+
+      def self.md_url(url, baseurl = "")
+        JekyllAeo::Utils::MdUrl.for(url, baseurl)
+      end
+
+      def self.build_custom_sections(eligible, section_defs)
+        section_defs.map do |section_def|
+          title = section_def["title"]
+          collection = section_def["collection"]
+
+          items = eligible.select do |item|
+            if collection.nil?
+              item[:collection].nil?
+            else
+              item[:collection] == collection
+            end
+          end
+
+          { title: title, items: items }
+        end
+      end
+
+      def self.build_auto_sections(eligible)
+        grouped = eligible.group_by { |item| item[:collection] }
+        sections = []
+
+        # Standalone pages first
+        sections << { title: "Pages", items: grouped.delete(nil) } if grouped.key?(nil)
+
+        # Collections alphabetically, "Optional" last
+        sorted_keys = grouped.keys.compact.sort_by do |key|
+          key == "optional" ? "zzz" : key
+        end
+
+        sorted_keys.each do |key|
+          sections << { title: titleize(key), items: grouped[key] }
+        end
+
+        sections
+      end
+
+      def self.titleize(label)
+        case label
+        when "posts"
+          "Blog Posts"
+        else
+          label.split(/[_-]/).map(&:capitalize).join(" ")
+        end
+      end
+
+      def self.build_llms_txt(site, sections, llms_config, config)
+        lines = []
+        lines << "# #{site.config['title']}"
+        lines << ""
+
+        description = llms_config["description"] || site.config["description"]
+        lines.push("> #{description}", "") if description && !description.to_s.empty?
+
+        append_full_txt_link(lines, site, config)
+        append_sections(lines, sections, llms_config, site)
+
+        "#{lines.join("\n").rstrip}\n"
+      end
+
+      def self.append_full_txt_link(lines, site, config)
+        full_txt_config = config["llms_full_txt"] || {}
+        return if full_txt_config["enabled"] == false
+
+        baseurl = site.config["baseurl"].to_s.chomp("/")
+        lines << "- [llms-full.txt](#{baseurl}/llms-full.txt): Complete contents of all pages"
+        lines << ""
+      end
+
+      def self.append_sections(lines, sections, llms_config, site)
+        sections.each do |section|
+          next if section[:items].empty?
+
+          lines << "## #{section[:title]}"
+          lines << ""
+
+          section[:items].each do |item|
+            url_md = md_url(item[:url], site.config["baseurl"])
+            entry = "- [#{item[:title]}](#{url_md})"
+            if llms_config["include_descriptions"] != false && !item[:description].empty?
+              entry += ": #{item[:description]}"
+            end
+            lines << entry
+          end
+
+          lines << ""
+        end
+      end
+
+      private_class_method :build_custom_sections, :build_auto_sections,
+                           :titleize, :build_llms_txt, :append_full_txt_link,
+                           :append_sections
+    end
+  end
+end

data/lib/jekyll-aeo/generators/markdown_page.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module JekyllAeo
+  module Generators
+    module MarkdownPage
+      YAML_FRONT_MATTER_REGEXP = /\A(---\s*\n.*?\n?)^(---\s*$\n?)/m
+
+      def self.process(obj, site)
+        config = JekyllAeo::Config.from_site(site)
+        dotmd_config = config["dotmd"]
+        source_path = JekyllAeo::Utils::SkipLogic.resolve_source_path(obj, site)
+        return if JekyllAeo::Utils::SkipLogic.skip?(obj, site, config)
+
+        dest_path = md_dest_path(obj, site)
+        body = extract_body(source_path, obj, dotmd_config)
+
+        if dotmd_config["include_last_modified"] && File.exist?(source_path)
+          last_modified = resolve_last_modified(obj, source_path)
+        end
+
+        if dotmd_config["dotmd_metadata"]
+          metadata = build_metadata_block(obj, site, config, last_modified)
+          header = build_header(obj, body, config, last_modified: nil)
+          result = metadata + header + body.lstrip
+        else
+          header = build_header(obj, body, config, last_modified: last_modified)
+          result = header + body.lstrip
+        end
+
+        result = result.gsub(/\n{3,}/, "\n\n")
+        result = "#{result.rstrip}\n"
+
+        FileUtils.mkdir_p(File.dirname(dest_path))
+        File.write(dest_path, result)
+
+        root_index = File.basename(dest_path) == "index.md" && File.dirname(dest_path) == site.dest
+        File.write(File.join(site.dest, "index.html.md"), result) if root_index
+      end
+
+      def self.extract_body(source_path, obj, dotmd_config)
+        if File.exist?(source_path)
+          raw = File.read(source_path, encoding: "utf-8")
+          body = raw.sub(YAML_FRONT_MATTER_REGEXP, "")
+          JekyllAeo::Utils::ContentStripper.strip(body, dotmd_config["md2dotmd"])
+        else
+          JekyllAeo::Utils::HtmlConverter.convert(obj.output, dotmd_config["html2dotmd"])
+        end
+      end
+
+      def self.build_header(obj, body, _config = nil, last_modified: nil)
+        header = ""
+        title = obj.data["title"]
+
+        header += "# #{title}\n\n" if title && !title.to_s.empty? && !body.lstrip.start_with?("# ")
+
+        description = obj.data["description"]
+        if description && !description.to_s.empty?
+          lines = description.to_s.split("\n")
+          header += if lines.length > 1
+                      "#{lines.map { |l| "> #{l}" }.join("\n")}\n\n"
+                    else
+                      "> #{description}\n\n"
+                    end
+        end
+
+        header += "> Last updated: #{last_modified}\n\n" if last_modified
+
+        header
+      end
+
+      def self.resolve_last_modified(obj, source_path)
+        lm = obj.data["last_modified_at"]
+        return format_date(lm) if lm
+
+        date = obj.data["date"]
+        return format_date(date) if date
+
+        return File.mtime(source_path).strftime("%Y-%m-%d") if File.exist?(source_path)
+
+        nil
+      end
+
+      def self.format_date(value)
+        case value
+        when Time, DateTime
+          value.strftime("%Y-%m-%d")
+        when String
+          begin
+            Date.parse(value).to_s
+          rescue StandardError
+            value
+          end
+        else
+          value.to_s
+        end
+      end
+
+      YAML_NEEDS_QUOTING = /[:\#"'{}\[\],&*?|<>=!%@`\n\r]/
+
+      def self.yaml_safe_scalar(value)
+        str = value.to_s
+        return str unless str.match?(YAML_NEEDS_QUOTING) || str.strip != str
+
+        escaped = str.gsub("\\", "\\\\\\\\").gsub('"', '\\"').gsub("\n", '\n')
+        "\"#{escaped}\""
+      end
+
+      SCALAR_FIELDS = %w[title description author lang].freeze
+
+      def self.build_metadata_block(obj, site, _config, last_modified)
+        lines = ["---"]
+        metadata_fields(obj, site, last_modified).each do |key, value|
+          next if value.nil? || value.to_s.empty?
+
+          lines << "#{key}: #{SCALAR_FIELDS.include?(key) ? yaml_safe_scalar(value) : value}"
+        end
+        lines << "---"
+        lines << ""
+        lines.join("\n")
+      end
+
+      def self.metadata_fields(obj, site, last_modified)
+        canonical = obj.data["canonical_url"] ||
+                    "#{site.config['url']}#{site.config['baseurl'].to_s.chomp('/')}#{obj.url}"
+        [
+          ["title", obj.data["title"]],
+          ["url", obj.url],
+          ["canonical", canonical],
+          ["description", obj.data["description"]],
+          ["author", obj.data["author"]],
+          ["date", obj.data["date"] ? format_date(obj.data["date"]) : nil],
+          ["last_modified", last_modified],
+          ["lang", obj.data["lang"] || obj.data["language"]]
+        ]
+      end
+
+      def self.md_dest_path(obj, site)
+        JekyllAeo::Utils::MdUrl.dest_path(obj, site)
+      end
+
+      private_class_method :build_header, :md_dest_path, :resolve_last_modified,
+                           :format_date, :build_metadata_block,
+                           :yaml_safe_scalar, :extract_body, :metadata_fields
+    end
+  end
+end