jekyll-agent-markdown 0.3.0 → 0.4.0

data/lib/jekyll/agent_markdown/document_header.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "jekyll"
+require_relative "configuration"
+
+module Jekyll
+  module AgentMarkdown
+    class DocumentHeader
+      URL_REQUIRED_MESSAGE =
+        "url is required for agent_markdown document headers; it must be an absolute HTTP(S) URL " \
+        "without credentials, a query, or a fragment"
+
+      def initialize(site, relative_url)
+        @site = site
+        @relative_url = relative_url
+      end
+
+      def prepend_to(document, body)
+        entries = ["# #{title(document)}"]
+        description = one_line(document.data["description"])
+        entries << description if description
+        entries << "Source: #{html_url(document)}"
+        "#{entries.join("\n\n")}\n\n---\n\n#{body}"
+      end
+
+      def html_url(document)
+        validate_site_url!(document)
+        "#{site.config["url"].to_s.sub(%r{/+\z}, "")}#{relative_url.call(document.url)}"
+      end
+
+      private
+
+      attr_reader :site, :relative_url
+
+      def title(document)
+        one_line(document.data["title"]) || one_line(basename(document)) || "Document"
+      end
+
+      def basename(document)
+        return document.basename_without_ext if document.respond_to?(:basename_without_ext)
+
+        document.basename
+      end
+
+      def one_line(value)
+        return unless value.is_a?(String)
+
+        normalized = value.gsub(/\s+/, " ").strip
+        normalized unless normalized.empty?
+      end
+
+      def validate_site_url!(document)
+        return if Configuration.absolute_http_url?(site.config["url"])
+
+        raise Jekyll::Errors::FatalException, "#{document.relative_path}: #{URL_REQUIRED_MESSAGE}"
+      end
+    end
+  end
+end

data/lib/jekyll/agent_markdown/document_settings.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+require "jekyll"
+require_relative "configuration"
+
+module Jekyll
+  module AgentMarkdown
+    class DocumentSettings
+      ALLOWED_SETTINGS = %w[export index section optional include_document_header].freeze
+
+      attr_reader :section
+
+      def initialize(document, settings, source_kind:, collection_name: nil)
+        @document = document
+        @settings = settings
+        @source_kind = SourceSettings.kind(source_kind, document_path)
+        @collection_name = SourceSettings.collection_name(collection_name, @source_kind, document_path)
+
+        apply(normalized_document_settings)
+      end
+
+      def export? = @export
+
+      def index? = @index
+
+      def optional? = @optional
+
+      def include_document_header? = @include_document_header
+
+      private
+
+      attr_reader :document, :settings, :source_kind, :collection_name
+
+      def normalized_document_settings
+        value = document.data.fetch("agent_markdown", true)
+        if value.is_a?(Hash)
+          return value.to_h { |key, setting| [key.to_s, setting] }.tap do |normalized|
+            validate_keys!(normalized)
+            validate_values!(normalized)
+          end
+        end
+
+        Configuration.enabled_value?(value, name: setting_name)
+        return { "export" => false, "index" => false } if Configuration.disabled?(value)
+
+        {}
+      end
+
+      def validate_keys!(document_settings)
+        unknown = document_settings.keys - ALLOWED_SETTINGS
+        return if unknown.empty?
+
+        suffix = "s" if unknown.length > 1
+        configuration_error("unknown agent_markdown setting#{suffix}: #{unknown.sort.join(", ")}")
+      end
+
+      def validate_values!(document_settings)
+        %w[export index optional include_document_header].each do |key|
+          next unless document_settings.key?(key)
+
+          Configuration.enabled_value?(document_settings.fetch(key), name: "agent_markdown.#{key} in #{document_path}")
+        end
+
+        return unless document_settings.key?("section")
+
+        section = document_settings.fetch("section")
+        return if section.is_a?(String) && !section.strip.empty?
+
+        configuration_error("agent_markdown.section must be a non-empty String; got #{section.inspect}")
+      end
+
+      def apply(document_settings)
+        @export, @index = export_status(document_settings)
+        @optional, @section = placement(document_settings)
+        @include_document_header = enabled_value(
+          document_settings,
+          "include_document_header",
+          Configuration.enabled?(settings, "include_document_header")
+        )
+      end
+
+      def export_status(document_settings)
+        source_enabled = source_enabled?
+        export = enabled_value(document_settings, "export", source_enabled)
+        index = enabled_value(document_settings, "index", source_enabled)
+        validate_export_and_index!(document_settings, index)
+        export &&= source_enabled
+        [export, export && index]
+      end
+
+      def validate_export_and_index!(document_settings, index)
+        return unless explicitly_disabled?(document_settings, "export") && document_settings.key?("index") && index
+
+        configuration_error("agent_markdown.export cannot be false when agent_markdown.index is true")
+      end
+
+      def explicitly_disabled?(document_settings, key)
+        document_settings.key?(key) && Configuration.disabled?(document_settings.fetch(key))
+      end
+
+      def placement(document_settings)
+        optional = enabled_value(document_settings, "optional", false)
+        section = SourceSettings.normalized_section(document_settings["section"])
+        if optional && section
+          configuration_error("agent_markdown.optional cannot be combined with agent_markdown.section")
+        end
+
+        [optional, optional ? "Optional" : section]
+      end
+
+      def enabled_value(document_settings, key, default)
+        return default unless document_settings.key?(key)
+
+        !Configuration.disabled?(document_settings.fetch(key))
+      end
+
+      def source_enabled?
+        case source_kind
+        when :post
+          Configuration.enabled?(settings, "posts")
+        when :page
+          Configuration.enabled?(settings, "pages")
+        when :collection
+          Configuration.collection_names(settings).include?(collection_name)
+        else
+          false
+        end
+      end
+
+      def setting_name = "agent_markdown in #{document_path}"
+
+      def document_path
+        return document.relative_path if document.respond_to?(:relative_path)
+        return document.path if document.respond_to?(:path)
+
+        "unknown document"
+      end
+
+      def configuration_error(message)
+        raise Jekyll::Errors::FatalException, "#{document_path}: #{message}"
+      end
+    end
+
+    module SourceSettings
+      SOURCE_KINDS = %i[post page collection].freeze
+
+      module_function
+
+      def kind(value, document_path)
+        source_kind = value.to_sym if value.respond_to?(:to_sym)
+        return source_kind if SOURCE_KINDS.include?(source_kind)
+
+        error(document_path, "unsupported source kind: #{value.inspect}")
+      end
+
+      def collection_name(value, source_kind, document_path)
+        return nil unless source_kind == :collection
+        return value.strip if value.is_a?(String) && !value.strip.empty?
+
+        error(document_path, "collection_name must be a non-empty String; got #{value.inspect}")
+      end
+
+      def normalized_section(value) = value&.gsub(/\s+/, " ")&.strip
+
+      def error(document_path, message)
+        raise Jekyll::Errors::FatalException, "#{document_path}: #{message}"
+      end
+    end
+  end
+end

data/lib/jekyll/agent_markdown/exported_document.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Jekyll
+  module AgentMarkdown
+    ExportedDocument = Data.define(
+      :source_document,
+      :source_kind,
+      :markdown_url,
+      :markdown_content,
+      :body,
+      :html_url,
+      :section,
+      :index,
+      :optional
+    )
+  end
+end

data/lib/jekyll/agent_markdown/generator.rb

@@ -1,72 +1,79 @@
 # frozen_string_literal: true
 
 require "jekyll"
+require_relative "author_metadata"
+require_relative "collection_validator"
 require_relative "configuration"
 require_relative "date_metadata"
 require_relative "destination_claims"
+require_relative "document_exporter"
+require_relative "llms_full_renderer"
 require_relative "llms_headings"
-require_relative "markdown_sibling_path"
-require_relative "raw_markdown_file"
+require_relative "llms_index_renderer"
+require_relative "llms_document_ordering"
+require_relative "metadata_footer"
+require_relative "source_documents"
 
 module Jekyll
   module AgentMarkdown
     class Generator < Jekyll::Generator
       include Jekyll::Filters::URLFilters
 
-      URL_REQUIRED_MESSAGE =
+      LLMS_TXT_URL_REQUIRED_MESSAGE =
         "url is required for agent_markdown.llms_txt and must be an absolute HTTP(S) URL " \
         "without credentials, a query, or a fragment"
+      LLMS_FULL_TXT_URL_REQUIRED_MESSAGE =
+        "url is required for agent_markdown.llms_full_txt and must be an absolute HTTP(S) URL " \
+        "without credentials, a query, or a fragment"
+      ONE_MEBIBYTE = 1_048_576
 
       # Run after normal- and high-priority generators so their post changes are exported.
+      # Destination claims therefore cover only what Jekyll knows at this point; a
+      # generator declaring `priority :lowest` runs later and can still claim the
+      # same destination. Jekyll offers no hook to detect that from here.
       priority :low
 
+      attr_reader :exported_documents
+
       def generate(site)
         settings = Configuration.for(site)
         return if settings == false
 
-        @context = Liquid::Context.new({}, {}, { site: site })
-        destination_claims = destination_claims(site)
-        included_posts = export_posts(site, settings, destination_claims) if Configuration.enabled?(settings, "posts")
-        write_llms_txt(site, settings, included_posts || [], destination_claims)
+        @exported_documents = exported_documents_for(site, settings)
+        write_llms_txt(site, settings, exported_documents, @destination_claims)
+        write_llms_full_txt(site, settings, exported_documents, @destination_claims)
       end
 
       private
 
-      def destination_claims(site)
-        DestinationClaims.new(site.dest).tap do |claims|
-          site.each_site_file { |item| claims.add_existing(item.destination(site.dest)) }
+      def exported_documents_for(site, settings)
+        @context = Liquid::Context.new({}, {}, { site: site })
+        collection_validator = CollectionValidator.new(site)
+        collection_validator.validate!(Configuration.collection_names(settings))
+        @destination_claims = destination_claims(site)
+        exporter = DocumentExporter.new(
+          site, settings, @destination_claims, content_for_post: ->(post) { post_content(site, post, settings) }
+        )
+        source_documents(site, settings, collection_validator).filter_map do |document, source_kind, collection_name|
+          exporter.export(document, source_kind:, collection_name:)
         end
       end
 
-      def export_posts(site, settings, destination_claims)
-        site.posts.docs.filter_map { |post| export_post(site, settings, destination_claims, post) }
+      def source_documents(site, settings, collection_validator)
+        SourceDocuments.new(site, settings, collection_validator:)
       end
 
-      def export_post(site, settings, destination_claims, post)
-        setting = post.data.fetch("agent_markdown", true)
-        setting_name = "agent_markdown in #{post.relative_path}"
-        return unless Configuration.enabled_value?(setting, name: setting_name)
-
-        file = RawMarkdownFile.new(site, MarkdownSiblingPath.for(post.url), post_content(post, settings))
-        url = file.url
-        return collision_warning(post, url) unless claim_destination?(destination_claims, site, file)
-
-        post.data["agent_markdown_url"] = url
-        site.static_files << file
-        post
-      end
-
-      def collision_warning(post, url)
-        Jekyll.logger.warn "AgentMarkdown:",
-                           "skipping #{post.relative_path}: #{url} already belongs to another file"
-        nil
+      def destination_claims(site)
+        DestinationClaims.new(site.dest).tap do |claims|
+          site.each_site_file { |item| claims.add_existing(item.destination(site.dest)) }
+        end
       end
 
-      def write_llms_txt(site, settings, posts, destination_claims)
+      def write_llms_txt(site, settings, documents, destination_claims)
         return unless Configuration.enabled?(settings, "llms_txt")
         return unless llms_txt_ready?(site, settings)
 
-        file = RawMarkdownFile.new(site, "/llms.txt", llms_txt(site, posts, settings))
+        file = RawMarkdownFile.new(site, "/llms.txt", LlmsIndexRenderer.new(site, settings, documents).to_s)
         return llms_txt_collision_warning unless claim_destination?(destination_claims, site, file)
 
         site.static_files << file
@@ -80,61 +87,62 @@ module Jekyll
         nil
       end
 
-      # A missing url only fails the build when llms_txt was explicitly
-      # configured; the default is to warn and skip so adding the gem never
-      # breaks a previously green build.
-      def llms_txt_ready?(site, settings)
-        return true if Configuration.absolute_http_url?(site.config["url"])
-        raise Jekyll::Errors::FatalException, URL_REQUIRED_MESSAGE if settings.key?("llms_txt")
+      def write_llms_full_txt(site, settings, documents, destination_claims)
+        return unless Configuration.enabled?(settings, "llms_full_txt")
 
-        Jekyll.logger.warn "AgentMarkdown:", "#{URL_REQUIRED_MESSAGE}; skipping llms.txt"
-        false
+        validate_llms_full_txt_url!(site)
+        placeholder = RawMarkdownFile.new(site, "/llms-full.txt", "")
+        return llms_full_txt_collision_warning unless claim_destination?(destination_claims, site, placeholder)
+
+        content = LlmsFullRenderer.new(site, settings, documents).to_s
+        warn_if_llms_full_txt_is_large(content)
+        file = RawMarkdownFile.new(site, "/llms-full.txt", content)
+
+        site.static_files << file
       end
 
-      def llms_txt(site, posts, settings)
-        sections = [LlmsHeadings.new(site, settings).to_s, article_links(site, posts, settings)]
-        "#{sections.reject(&:empty?).join("\n\n")}\n"
+      def validate_llms_full_txt_url!(site)
+        return if Configuration.absolute_http_url?(site.config["url"])
+
+        raise Jekyll::Errors::FatalException, LLMS_FULL_TXT_URL_REQUIRED_MESSAGE
       end
 
-      def article_links(site, posts, settings)
-        site_url = site.config["url"].sub(%r{/+\z}, "")
-        sorted_posts(posts, settings).map { |post| article_link(site_url, post, settings) }.join("\n")
+      def warn_if_llms_full_txt_is_large(content)
+        return unless content.bytesize > ONE_MEBIBYTE
+
+        Jekyll.logger.warn "AgentMarkdown:", "llms-full.txt exceeds 1 MiB; consider reducing its size"
       end
 
-      def sorted_posts(posts, settings)
-        dated, undated = posts.map { |post| [post, date_metadata(post).published_date] }.partition(&:last)
-        dated.sort_by!(&:last)
-        dated.reverse! if Configuration.sort_order(settings) == "desc"
-        (dated + undated).map(&:first)
+      def llms_full_txt_collision_warning
+        Jekyll.logger.warn "AgentMarkdown:",
+                           "skipping /llms-full.txt: the destination already belongs to another file"
+        nil
       end
 
-      def article_link(site_url, post, settings)
-        url = "#{site_url}#{relative_url(post.data.fetch("agent_markdown_url"))}"
-        link = "- [#{link_title(post)}](#{escaped_link_url(url)})"
-        return link unless Configuration.enabled?(settings, "include_dates")
+      # A missing url only fails the build when llms_txt was explicitly
+      # configured; the default is to warn and skip so adding the gem never
+      # breaks a previously green build.
+      def llms_txt_ready?(site, settings)
+        return true if Configuration.absolute_http_url?(site.config["url"])
+        raise Jekyll::Errors::FatalException, LLMS_TXT_URL_REQUIRED_MESSAGE if settings.key?("llms_txt")
 
-        [link, date_metadata(post).to_s].reject(&:empty?).join(" | ")
+        Jekyll.logger.warn "AgentMarkdown:", "#{LLMS_TXT_URL_REQUIRED_MESSAGE}; skipping llms.txt"
+        false
       end
 
-      def post_content(post, settings)
-        return post.content unless Configuration.enabled?(settings, "include_dates")
-
-        date_metadata(post).append_to(post.content)
+      def post_content(site, post, settings)
+        entries = []
+        entries << date_metadata(post).to_s if Configuration.enabled?(settings, "include_dates")
+        entries << AuthorMetadata.new(site.config).to_s if Configuration.enabled?(settings, "include_author")
+        MetadataFooter.new(entries).append_to(post.content)
       end
 
       def date_metadata(post) = DateMetadata.new(post.data)
 
-      # Backslashes and square brackets would end the Markdown link text early;
-      # whitespace runs (including newlines) would break the one-entry-per-line
-      # format.
-      def link_title(post)
-        post.data.fetch("title", post.basename_without_ext)
-            .to_s.gsub(/\s+/, " ").strip
-            .gsub(/[\\\[\]]/) { |character| "\\#{character}" }
+      # Retained for the public test seam used by existing integrations.
+      def sorted_posts(posts, settings)
+        LlmsDocumentOrdering.new(settings).legacy_ordered(posts) { |post| date_metadata(post).published_date }
       end
-
-      # Unescaped parentheses would end the Markdown link destination early.
-      def escaped_link_url(url) = url.gsub("(", "%28").gsub(")", "%29")
     end
   end
 end

data/lib/jekyll/agent_markdown/llms_document_index.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require_relative "configuration"
+require_relative "llms_text"
+
+module Jekyll
+  module AgentMarkdown
+    class LlmsDocumentIndex
+      Section = Data.define(:name, :documents)
+      OPTIONAL_KEY = "optional"
+
+      def initialize(settings, documents)
+        @settings = settings
+        @documents = documents.select(&:index)
+      end
+
+      def default_compatibility?
+        Configuration.enabled?(settings, "posts") &&
+          !Configuration.enabled?(settings, "pages") &&
+          Configuration.collection_names(settings).empty? &&
+          !Configuration.enabled?(settings, "include_descriptions") &&
+          documents.none? { |document| document.optional || document.section }
+      end
+
+      def sections
+        @sections ||= build_sections
+      end
+
+      private
+
+      attr_reader :settings, :documents
+
+      def build_sections
+        groups, names = grouped_documents
+        section_registry(groups, names).filter_map do |key, name|
+          Section.new(name, groups.fetch(key)) if groups.key?(key)
+        end
+      end
+
+      def grouped_documents
+        documents.each_with_object([Hash.new { |hash, key| hash[key] = [] }, {}]) do |document, entries|
+          groups, names = entries
+          key = resolved_section_key(document)
+          groups[key] << document
+          names[key] ||= section_name(document, key)
+        end
+      end
+
+      def section_registry(groups, names)
+        registry = default_registry.merge(custom_registry(groups, names))
+        if groups.key?(OPTIONAL_KEY)
+          registry.delete(OPTIONAL_KEY)
+          registry[OPTIONAL_KEY] = "Optional"
+        end
+        registry
+      end
+
+      def default_registry
+        @default_registry ||= default_section_names.each_with_object({}) do |name, registry|
+          registry[section_key(name)] ||= name
+        end
+      end
+
+      def custom_registry(groups, names)
+        groups.each_with_object({}) do |(key, _documents), registry|
+          registry[key] = names.fetch(key) if custom_section?(key)
+        end
+      end
+
+      def custom_section?(key) = key != OPTIONAL_KEY && !default_registry.key?(key)
+
+      def default_section_names
+        @default_section_names ||= %w[Articles Pages] + collection_section_names
+      end
+
+      def collection_section_names
+        Configuration.collection_names(settings).map { |name| humanize(name) }
+      end
+
+      def resolved_section_key(document)
+        return OPTIONAL_KEY if document.optional || section_key(document.section) == OPTIONAL_KEY
+        return section_key(document.section) if document.section
+
+        implicit_section_key(document)
+      end
+
+      def section_name(document, key)
+        return "Optional" if key == OPTIONAL_KEY
+        return document.section if document.section
+
+        default_registry.fetch(key)
+      end
+
+      def implicit_section_key(document)
+        case document.source_kind
+        when :post
+          section_key("Articles")
+        when :page
+          section_key("Pages")
+        when :collection
+          section_key(humanize(collection_label(document)))
+        end
+      end
+
+      def section_key(name)
+        LlmsText.one_line(name).downcase.split(/[-_\s]+/).reject(&:empty?).join(" ")
+      end
+
+      def collection_label(document)
+        return unless document.source_kind == :collection
+
+        collection = document.source_document.collection
+        collection.label if collection.respond_to?(:label)
+      end
+
+      def humanize(name)
+        name.split(/[-_\s]+/).filter_map do |word|
+          next if word.empty?
+
+          "#{word[0].upcase}#{word[1..].to_s.downcase}"
+        end.join(" ")
+      end
+    end
+  end
+end

data/lib/jekyll/agent_markdown/llms_document_ordering.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative "configuration"
+
+module Jekyll
+  module AgentMarkdown
+    class LlmsDocumentOrdering
+      def initialize(settings)
+        @settings = settings
+      end
+
+      def ordered(documents, &published_date)
+        dated, undated = records(documents, &published_date).partition { |_document, date, _order| !date.nil? }
+        dated.sort! { |left, right| compare(left, right) }
+        (dated + undated).map(&:first)
+      end
+
+      def legacy_ordered(documents, &published_date)
+        dated, undated = records(documents, &published_date).partition { |_document, date, _order| !date.nil? }
+        dated.sort_by! { |_document, date, _order| date }
+        dated.reverse! if Configuration.sort_order(settings) == "desc"
+        (dated + undated).map(&:first)
+      end
+
+      private
+
+      attr_reader :settings
+
+      def records(documents)
+        documents.each_with_index.map { |document, order| [document, yield(document), order] }
+      end
+
+      def compare(left, right)
+        comparison = left[1] <=> right[1]
+        comparison = -comparison if Configuration.sort_order(settings) == "desc"
+        comparison.zero? ? left[2] <=> right[2] : comparison
+      end
+    end
+  end
+end