jekyll-agent-markdown 0.3.0 → 0.4.0

data/examples/nginx/negotiation.js

@@ -0,0 +1,267 @@
+const HTML = {
+    name: "html",
+    contentType: "text/html; charset=utf-8",
+    type: "text",
+    subtype: "html",
+    parameters: {charset: "utf-8"}
+};
+const MARKDOWN = {
+    name: "markdown",
+    contentType: "text/markdown; charset=utf-8",
+    type: "text",
+    subtype: "markdown",
+    parameters: {charset: "utf-8"}
+};
+const HOP_BY_HOP = {
+    connection: true,
+    "keep-alive": true,
+    "proxy-authenticate": true,
+    "proxy-authorization": true,
+    "proxy-connection": true,
+    te: true,
+    trailer: true,
+    "transfer-encoding": true,
+    upgrade: true
+};
+const SAFE_METHODS = {GET: true, HEAD: true};
+
+async function serve(r) {
+    // njs subrequests are always GET, so a POST or DELETE reaching this handler
+    // would be silently downgraded into a static read of the page body.
+    if (!SAFE_METHODS[r.method]) {
+        r.headersOut.Allow = "GET, HEAD";
+        r.headersOut["Content-Type"] = "text/plain; charset=utf-8";
+        r.return(405, "Method Not Allowed\n");
+        return;
+    }
+
+    const explicitMarkdown = r.uri.endsWith(".md");
+    const preference = explicitMarkdown
+        ? {selected: "markdown", htmlAcceptable: false}
+        : negotiate(r.headersIn.Accept);
+
+    if (!preference.selected) {
+        r.headersOut["Content-Type"] = "text/plain; charset=utf-8";
+        r.headersOut.Vary = mergeVary(r.headersOut.Vary, "Accept");
+        r.return(406, "Not Acceptable\n");
+        return;
+    }
+
+    const paths = variantPaths(r.uri, explicitMarkdown);
+    const variant = await fetchVariant(r, paths, preference, explicitMarkdown);
+    const reply = variant.reply;
+
+    copyResponseHeaders(r, reply);
+    if (!explicitMarkdown) r.headersOut.Vary = mergeVary(reply.headersOut.Vary, "Accept");
+
+    if (reply.status < 200 || reply.status >= 300) {
+        r.return(reply.status, reply.responseBuffer);
+        return;
+    }
+
+    const representation = variant.served === "markdown" ? MARKDOWN : HTML;
+    r.headersOut["Content-Type"] = representation.contentType;
+    r.headersOut.Link = mergeLink(reply.headersOut.Link, alternateLink(variant.served, paths));
+    r.return(reply.status, reply.responseBuffer);
+}
+
+// A negotiated Markdown variant is not guaranteed to exist: the plugin exports
+// posts by default, so most page URLs have no .md sibling until the site opts
+// pages or collections in. Fall back to HTML when the client accepts it rather
+// than turning a page the host could serve into a 404.
+async function fetchVariant(r, paths, preference, explicitMarkdown) {
+    const reply = await r.subrequest(`/_jekyll_asset${paths[preference.selected]}`);
+    if (preference.selected !== "markdown" || explicitMarkdown) {
+        return {reply, served: preference.selected};
+    }
+    if (reply.status !== 404 || !preference.htmlAcceptable) return {reply, served: "markdown"};
+
+    const fallback = await r.subrequest(`/_jekyll_asset${paths.html}`);
+    if (fallback.status === 404) return {reply, served: "markdown"};
+
+    return {reply: fallback, served: "html"};
+}
+
+function negotiate(accept) {
+    if (!accept || !accept.trim()) return {selected: "html", htmlAcceptable: true};
+
+    const ranges = parseAccept(accept);
+    const html = qualityFor(HTML, ranges);
+    const markdown = qualityFor(MARKDOWN, ranges);
+    const htmlAcceptable = html.quality > 0;
+
+    if (html.quality <= 0 && markdown.quality <= 0) return {selected: null, htmlAcceptable};
+    if (markdown.quality > html.quality) return {selected: "markdown", htmlAcceptable};
+    if (html.quality > markdown.quality) return {selected: "html", htmlAcceptable};
+
+    const explicitMarkdown = ranges.some((range) =>
+        range.type === "text" &&
+        range.subtype === "markdown" &&
+        range.quality > 0 &&
+        specificity(range, MARKDOWN) !== null
+    );
+    return {selected: explicitMarkdown ? "markdown" : "html", htmlAcceptable};
+}
+
+function qualityFor(representation, ranges) {
+    let bestSpecificity = -1;
+    let quality = 0;
+
+    for (const range of ranges) {
+        const rangeSpecificity = specificity(range, representation);
+        if (rangeSpecificity === null || rangeSpecificity < bestSpecificity) continue;
+
+        if (rangeSpecificity > bestSpecificity) {
+            bestSpecificity = rangeSpecificity;
+            quality = range.quality;
+        } else {
+            quality = Math.max(quality, range.quality);
+        }
+    }
+
+    return {quality, specificity: bestSpecificity};
+}
+
+function specificity(range, representation) {
+    if (range.type === "*" && range.subtype !== "*") return null;
+    if (range.type !== "*" && range.type !== representation.type) return null;
+    if (range.subtype !== "*" && range.subtype !== representation.subtype) return null;
+
+    for (const name in range.parameters) {
+        if (representation.parameters[name] !== range.parameters[name]) return null;
+    }
+
+    const mediaSpecificity = range.type === "*" ? 0 : range.subtype === "*" ? 1 : 2;
+    return (mediaSpecificity * 100) + Object.keys(range.parameters).length;
+}
+
+function parseAccept(accept) {
+    return splitQuoted(accept, ",").map(parseRange).filter((range) => range !== null);
+}
+
+function parseRange(source) {
+    const parts = splitQuoted(source, ";");
+    const media = parts.shift().trim().toLowerCase();
+    const match = media.match(/^([^/\s]+)\/([^/\s]+)$/);
+    if (!match) return null;
+
+    const range = {type: match[1], subtype: match[2], quality: 1, parameters: {}};
+    for (const sourceParameter of parts) {
+        const separator = sourceParameter.indexOf("=");
+        if (separator < 1) {
+            if (sourceParameter.trim().toLowerCase() === "q") range.quality = 0;
+            continue;
+        }
+
+        const name = sourceParameter.slice(0, separator).trim().toLowerCase();
+        const rawValue = sourceParameter.slice(separator + 1).trim();
+        if (name === "q") {
+            range.quality = parseQuality(rawValue.toLowerCase());
+        } else {
+            range.parameters[name] = unquote(rawValue).toLowerCase();
+        }
+    }
+
+    return range;
+}
+
+function parseQuality(value) {
+    return /^(?:0(?:\.\d{0,3})?|1(?:\.0{0,3})?)$/.test(value) ? Number(value) : 0;
+}
+
+function splitQuoted(source, separator) {
+    const parts = [];
+    let current = "";
+    let quoted = false;
+    let escaped = false;
+
+    for (const character of source) {
+        if (escaped) {
+            current += character;
+            escaped = false;
+        } else if (character === "\\" && quoted) {
+            current += character;
+            escaped = true;
+        } else if (character === '"') {
+            current += character;
+            quoted = !quoted;
+        } else if (character === separator && !quoted) {
+            parts.push(current);
+            current = "";
+        } else {
+            current += character;
+        }
+    }
+
+    parts.push(current);
+    return parts;
+}
+
+function unquote(value) {
+    if (value.length >= 2 && value.startsWith('"') && value.endsWith('"')) {
+        return value.slice(1, -1).replace(/\\(.)/g, "$1");
+    }
+    return value;
+}
+
+function variantPaths(pathname, explicitMarkdown) {
+    if (explicitMarkdown) return {html: htmlPathFor(pathname), markdown: pathname};
+    return {html: pathname, markdown: markdownPathFor(pathname)};
+}
+
+function markdownPathFor(pathname) {
+    if (pathname === "/") return "/index.md";
+    if (pathname.endsWith("/")) return `${pathname.slice(0, -1)}.md`;
+    if (/\.html?$/.test(pathname)) return pathname.replace(/\.html?$/, ".md");
+    return `${pathname}.md`;
+}
+
+function htmlPathFor(pathname) {
+    if (pathname === "/index.md") return "/";
+    return `${pathname.slice(0, -3)}/`;
+}
+
+function alternateLink(selected, paths) {
+    const alternate = selected === "markdown" ? "html" : "markdown";
+    return `<${paths[alternate]}>; rel="alternate"; type="text/${alternate}"`;
+}
+
+function mergeVary(current, field) {
+    const fields = (current || "").split(",").map((value) => value.trim()).filter(Boolean);
+    if (!fields.some((value) => value.toLowerCase() === field.toLowerCase())) fields.push(field);
+    return fields.join(", ");
+}
+
+function mergeLink(current, alternate) {
+    if (!current || current.includes(alternate)) return current || alternate;
+    return `${current}, ${alternate}`;
+}
+
+function copyResponseHeaders(r, reply) {
+    const excluded = connectionOptions(reply.headersOut);
+    for (const name in HOP_BY_HOP) excluded[name] = true;
+    excluded["content-length"] = true;
+
+    for (const name in reply.headersOut) {
+        if (excluded[name.toLowerCase()]) continue;
+        r.headersOut[name] = reply.headersOut[name];
+    }
+}
+
+function connectionOptions(headers) {
+    const options = {};
+    for (const name in headers) {
+        if (name.toLowerCase() !== "connection") continue;
+
+        const values = Array.isArray(headers[name]) ? headers[name] : [headers[name]];
+        for (const value of values) {
+            for (const token of String(value).split(",")) {
+                const option = token.trim().toLowerCase();
+                if (option) options[option] = true;
+            }
+        }
+    }
+    return options;
+}
+
+export default {serve};

data/examples/nginx/nginx.conf

@@ -0,0 +1,40 @@
+load_module modules/ngx_http_js_module.so;
+
+events {}
+
+http {
+    # Without these the pass-through location below labels CSS, JavaScript,
+    # fonts, and images as text/plain, which stops stylesheets and modules
+    # from loading in browsers.
+    include mime.types;
+    default_type application/octet-stream;
+
+    js_engine qjs;
+    js_import negotiation from /etc/nginx/njs/negotiation.js;
+    subrequest_output_buffer_size 10m;
+
+    server {
+        listen 8080;
+        server_name _;
+        root /srv/jekyll/_site;
+        index index.html;
+
+        location ^~ /_jekyll_asset/ {
+            internal;
+            alias /srv/jekyll/_site/;
+            index index.html;
+        }
+
+        location ~* \.(?:md|html?)$ {
+            js_content negotiation.serve;
+        }
+
+        location ~ \.[^/]+$ {
+            try_files $uri =404;
+        }
+
+        location / {
+            js_content negotiation.serve;
+        }
+    }
+}

data/lib/jekyll/agent_markdown/agent_markdown_link_tag.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "cgi"
+require "jekyll"
+
+module Jekyll
+  module AgentMarkdown
+    class AgentMarkdownLinkTag < Liquid::Tag
+      include Jekyll::Filters::URLFilters
+
+      def render(context)
+        @context = context
+        markdown_url = context["page"]&.[]("agent_markdown_url")
+        return "" unless markdown_url
+
+        %(<link rel="alternate" type="text/markdown" href="#{CGI.escapeHTML(relative_url(markdown_url))}">)
+      end
+    end
+  end
+end

data/lib/jekyll/agent_markdown/author_metadata.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative "llms_text"
+
+module Jekyll
+  module AgentMarkdown
+    class AuthorMetadata
+      def initialize(config)
+        @config = config
+      end
+
+      def to_s
+        author_name = name
+        author_name.empty? ? "" : "Author: #{author_name}"
+      end
+
+      private
+
+      attr_reader :config
+
+      def name
+        author = config["author"]
+        author = author["name"] || author[:name] if author.is_a?(Hash)
+        LlmsText.one_line(author)
+      end
+    end
+  end
+end

data/lib/jekyll/agent_markdown/collection_validator.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "jekyll"
+
+module Jekyll
+  module AgentMarkdown
+    class CollectionValidator
+      def initialize(site)
+        @site = site
+      end
+
+      def validate!(collection_names)
+        collection_names.each { |collection_name| collection!(collection_name) }
+      end
+
+      def collection!(collection_name)
+        collection = site.collections[collection_name]
+        return collection if collection&.write? && valid_document_urls?(collection, collection_name)
+
+        raise_collection_error(collection, collection_name)
+      end
+
+      private
+
+      attr_reader :site
+
+      def valid_document_urls?(collection, collection_name)
+        collection.docs.all? do |document|
+          !markdown_document?(document) || public_url?(document, collection_name)
+        end
+      end
+
+      def markdown_document?(document)
+        document.is_a?(Jekyll::Document) && markdown_extension?(document.extname)
+      end
+
+      def markdown_extension?(extension)
+        site.config.fetch("markdown_ext").split(",").any? do |markdown_extension|
+          ".#{markdown_extension.strip.downcase}" == extension.to_s.downcase
+        end
+      end
+
+      def public_url?(document, collection_name)
+        url = document.url
+        return true if url.is_a?(String) && url.start_with?("/") && !url.start_with?("//")
+
+        public_url_error(document, collection_name)
+      rescue Jekyll::Errors::FatalException
+        raise
+      rescue StandardError => e
+        public_url_error(document, collection_name, e.message)
+      end
+
+      def public_url_error(document, collection_name, detail = nil)
+        message = "#{document.relative_path}: configured collection #{collection_name.inspect} " \
+                  "cannot produce a public document URL"
+        message = "#{message} (#{detail})" if detail
+        raise Jekyll::Errors::FatalException,
+              message
+      end
+
+      def raise_collection_error(collection, collection_name)
+        if collection.nil?
+          raise Jekyll::Errors::FatalException,
+                "agent_markdown.collections includes #{collection_name.inspect}, but that collection does not exist"
+        end
+
+        raise Jekyll::Errors::FatalException,
+              "agent_markdown.collections includes #{collection_name.inspect}, but its output is disabled"
+      end
+    end
+  end
+end

data/lib/jekyll/agent_markdown/configuration.rb

@@ -6,11 +6,25 @@ require "uri"
 module Jekyll
   module AgentMarkdown
     class Configuration
-      ALLOWED_SETTINGS = %w[include_author include_dates llms_txt posts sort].freeze
+      DEFAULTS = {
+        "posts" => true,
+        "pages" => false,
+        "collections" => [].freeze,
+        "llms_txt" => true,
+        "llms_full_txt" => false,
+        "include_descriptions" => false,
+        "include_document_header" => false,
+        "include_author" => true,
+        "include_dates" => true,
+        "sort" => "desc"
+      }.freeze
+      ALLOWED_SETTINGS = DEFAULTS.keys.freeze
       FALSE_STRINGS = %w[false no off].freeze
       SORT_ORDERS = %w[asc desc].freeze
 
       class << self
+        def defaults = DEFAULTS.dup
+
         def for(site)
           settings = site.config["agent_markdown"]
           return {} if settings.nil? || settings == true
@@ -24,11 +38,15 @@ module Jekyll
         end
 
         def enabled?(settings, key)
-          !disabled?(settings.fetch(key, true))
+          !disabled?(settings.fetch(key, DEFAULTS.fetch(key)))
         end
 
         def sort_order(settings)
-          settings.fetch("sort", "desc")
+          settings.fetch("sort", DEFAULTS.fetch("sort"))
+        end
+
+        def collection_names(settings)
+          settings.fetch("collections", DEFAULTS.fetch("collections"))
         end
 
         def enabled_value?(value, name:)
@@ -56,7 +74,7 @@ module Jekyll
         def normalized_settings(settings)
           normalized = settings.to_h { |key, value| [key.to_s, value] }
           validate_keys!(normalized)
-          validate_values!(normalized)
+          normalize_values!(normalized)
           normalized
         end
 
@@ -69,10 +87,13 @@ module Jekyll
                 "unknown agent_markdown setting#{suffix}: #{unknown.sort.join(", ")}"
         end
 
-        def validate_values!(settings)
+        def normalize_values!(settings)
           settings.each do |key, value|
-            if key == "sort"
+            case key
+            when "sort"
               validate_sort_order!(value)
+            when "collections"
+              settings[key] = CollectionNames.normalize(value)
             else
               validate_value!("agent_markdown.#{key}", value)
             end
@@ -97,6 +118,46 @@ module Jekyll
           value == true || disabled?(value)
         end
       end
+
+      module CollectionNames
+        module_function
+
+        def normalize(value)
+          validate_array!(value)
+          names = value.map { |name| normalized_name!(name) }
+          validate_unique!(names)
+          validate_reserved!(names)
+          names
+        end
+
+        def validate_array!(value)
+          return if value.is_a?(Array)
+
+          raise Jekyll::Errors::FatalException,
+                "agent_markdown.collections must be an Array of unique, non-empty collection names; " \
+                "got #{value.inspect}"
+        end
+
+        def normalized_name!(name)
+          return name.strip if name.is_a?(String) && !name.strip.empty?
+
+          raise Jekyll::Errors::FatalException,
+                "agent_markdown.collections must contain only non-empty collection names; got #{name.inspect}"
+        end
+
+        def validate_unique!(names)
+          return if names.uniq.length == names.length
+
+          raise Jekyll::Errors::FatalException, "agent_markdown.collections must contain unique collection names"
+        end
+
+        def validate_reserved!(names)
+          return unless names.include?("posts")
+
+          raise Jekyll::Errors::FatalException,
+                "agent_markdown.collections cannot include \"posts\"; use agent_markdown.posts instead"
+        end
+      end
     end
   end
 end

data/lib/jekyll/agent_markdown/date_metadata.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "date"
+require_relative "metadata_footer"
 
 module Jekyll
   module AgentMarkdown
@@ -21,10 +22,7 @@ module Jekyll
       end
 
       def append_to(content)
-        return content if to_s.empty?
-        return "#{self}\n" if content.empty?
-
-        "#{content}#{separator_for(content)}---\n#{self}\n"
+        MetadataFooter.new([to_s]).append_to(content)
       end
 
       def published_date = parsed_date(@data["date"])
@@ -36,10 +34,6 @@ module Jekyll
         "#{label}: #{date}" if date
       end
 
-      def separator_for(content)
-        content.end_with?("\n") ? "\n" : "\n\n"
-      end
-
       def formatted_date(value)
         parsed_date(value)&.strftime(DATE_FORMAT)
       end

data/lib/jekyll/agent_markdown/document_exporter.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "jekyll"
+require_relative "configuration"
+require_relative "document_header"
+require_relative "document_settings"
+require_relative "exported_document"
+require_relative "markdown_sibling_path"
+require_relative "raw_markdown_file"
+
+module Jekyll
+  module AgentMarkdown
+    class DocumentExporter
+      include Jekyll::Filters::URLFilters
+
+      # The published bytes and the same document without the generated header.
+      # llms-full.txt renders its own heading and Source line, so inlining the
+      # published bytes there would repeat both and nest a second level-one
+      # heading under the section.
+      Contents = Data.define(:content, :body)
+
+      def initialize(site, settings, destination_claims, content_for_post:)
+        @site = site
+        @settings = settings
+        @destination_claims = destination_claims
+        @content_for_post = content_for_post
+        @context = Liquid::Context.new({}, {}, { site: site })
+        @header = DocumentHeader.new(site, method(:relative_url))
+      end
+
+      def export(document, source_kind:, collection_name: nil)
+        # Opted-out and collided documents must not keep a stale or authored
+        # value: the alternate-link tag reads it as proof of publication.
+        document.data.delete("agent_markdown_url")
+        document_settings = DocumentSettings.new(
+          document, settings, source_kind: source_kind, collection_name: collection_name
+        )
+        return unless document_settings.export?
+
+        claim_export(document, source_kind, document_settings)
+      end
+
+      private
+
+      attr_reader :site, :settings, :destination_claims, :content_for_post, :header
+
+      def claim_export(document, source_kind, document_settings)
+        file, contents = markdown_file(document, source_kind, document_settings)
+        return collision_warning(document, file.url) unless destination_claims.claim?(file.destination(site.dest))
+
+        publish(document, source_kind, document_settings, file, contents)
+      end
+
+      def markdown_file(document, source_kind, document_settings)
+        contents = document_contents(document, source_kind, document_settings)
+        file = RawMarkdownFile.new(site, MarkdownSiblingPath.for(document.url), contents.content)
+        [file, contents]
+      end
+
+      def publish(document, source_kind, document_settings, file, contents)
+        document.data["agent_markdown_url"] = file.url
+        site.static_files << file
+        exported_document(document, source_kind, document_settings, file, contents)
+      end
+
+      def exported_document(document, source_kind, document_settings, file, contents)
+        ExportedDocument.new(
+          source_document: document,
+          source_kind: source_kind,
+          markdown_url: file.url,
+          markdown_content: contents.content,
+          body: contents.body,
+          html_url: html_url(document, document_settings),
+          **placement(document_settings)
+        )
+      end
+
+      def placement(document_settings)
+        {
+          section: document_settings.section,
+          index: document_settings.index?,
+          optional: document_settings.optional?
+        }
+      end
+
+      def document_contents(document, source_kind, document_settings)
+        body = source_kind == :post ? content_for_post.call(document) : document.content.to_s
+        return Contents.new(content: body, body: body) unless document_settings.include_document_header?
+
+        Contents.new(content: header.prepend_to(document, body), body: body)
+      end
+
+      def html_url(document, document_settings)
+        return unvalidated_html_url(document) unless document_settings.include_document_header?
+
+        header.html_url(document)
+      end
+
+      def unvalidated_html_url(document)
+        path = relative_url(document.url)
+        return path unless Configuration.absolute_http_url?(site.config["url"])
+
+        "#{site.config["url"].sub(%r{/+\z}, "")}#{path}"
+      end
+
+      def collision_warning(document, url)
+        Jekyll.logger.warn "AgentMarkdown:",
+                           "skipping #{document.relative_path}: #{url} already belongs to another file"
+        nil
+      end
+    end
+  end
+end