jekyll-agent-markdown 0.3.0 → 0.4.0

data/lib/jekyll/agent_markdown/llms_full_renderer.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "jekyll"
+require_relative "configuration"
+require_relative "date_metadata"
+require_relative "llms_document_index"
+require_relative "llms_document_ordering"
+require_relative "llms_text"
+
+module Jekyll
+  module AgentMarkdown
+    class LlmsFullRenderer
+      include Jekyll::Filters::URLFilters
+
+      def initialize(site, settings, documents)
+        @site = site
+        @settings = settings
+        @index = LlmsDocumentIndex.new(settings, documents)
+        @context = Liquid::Context.new({}, {}, { site: site })
+      end
+
+      def to_s
+        sections = index.sections.map { |section| render_section(section) }
+        "#{([site_heading] + sections).join("\n\n")}\n"
+      end
+
+      private
+
+      attr_reader :site, :settings, :index
+
+      def site_heading
+        "# #{LlmsText.one_line(site.config.fetch("title", ""))}".rstrip
+      end
+
+      def render_section(section)
+        (["## #{LlmsText.literal(section.name)}"] + sorted(section.documents).map do |document|
+          render_document(document)
+        end)
+          .join("\n\n")
+      end
+
+      def render_document(document)
+        ["### [#{title(document)}](#{escaped_link_url(absolute_markdown_url(document))})",
+         "Source: #{absolute_html_url(document)}", document.body].join("\n\n")
+      end
+
+      def sorted(documents)
+        ordering = LlmsDocumentOrdering.new(settings)
+        dates = proc { |document| published_date(document) }
+        return ordering.legacy_ordered(documents, &dates) if index.default_compatibility?
+
+        ordering.ordered(documents, &dates)
+      end
+
+      def absolute_markdown_url(document)
+        "#{site.config.fetch("url").sub(%r{/+\z}, "")}#{relative_url(document.markdown_url)}"
+      end
+
+      def absolute_html_url(document)
+        return document.html_url if document.html_url.start_with?("http://", "https://")
+
+        "#{site.config.fetch("url").sub(%r{/+\z}, "")}#{relative_url(document.source_document.url)}"
+      end
+
+      def title(document) = LlmsText.link_title(document.source_document)
+
+      def published_date(document) = DateMetadata.new(document.source_document.data).published_date
+
+      def escaped_link_url(url) = LlmsText.escaped_link_url(url)
+    end
+  end
+end

data/lib/jekyll/agent_markdown/llms_headings.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
+require_relative "author_metadata"
 require_relative "configuration"
+require_relative "llms_text"
 
 module Jekyll
   module AgentMarkdown
@@ -10,38 +12,31 @@ module Jekyll
         @settings = settings
       end
 
-      def to_s
-        [title_heading, description_heading, author_heading, "## Articles", articles_description]
-          .compact.join("\n\n")
+      def to_s(include_articles: true)
+        headings = [title_heading, description_heading, author_heading]
+        headings += ["## Articles", articles_description] if include_articles
+        headings.compact.join("\n\n")
       end
 
       private
 
       attr_reader :site, :settings
 
-      def title_heading = "# #{one_line(site.config.fetch("title", ""))}".rstrip
+      def title_heading = "# #{LlmsText.one_line(site.config.fetch("title", ""))}".rstrip
 
       def description_heading
-        description = one_line(site.config["description"])
+        description = LlmsText.one_line(site.config["description"])
         "> #{description}" unless description.empty?
       end
 
       def author_heading
         return unless Configuration.enabled?(settings, "include_author")
 
-        name = author_name
-        "Author: #{name}" unless name.empty?
-      end
-
-      def author_name
-        author = site.config["author"]
-        author = author["name"] || author[:name] if author.is_a?(Hash)
-        one_line(author)
+        metadata = AuthorMetadata.new(site.config).to_s
+        metadata unless metadata.empty?
       end
 
       def articles_description = "> Posts only. Pages and collections are not included."
-
-      def one_line(value) = value ? value.to_s.gsub(/\s+/, " ").strip : ""
     end
   end
 end

data/lib/jekyll/agent_markdown/llms_index_renderer.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "cgi"
+require "jekyll"
+require_relative "configuration"
+require_relative "date_metadata"
+require_relative "llms_document_index"
+require_relative "llms_document_ordering"
+require_relative "llms_headings"
+require_relative "llms_text"
+
+module Jekyll
+  module AgentMarkdown
+    class LlmsIndexRenderer
+      include Jekyll::Filters::URLFilters
+      include Liquid::StandardFilters
+
+      def initialize(site, settings, documents)
+        @site = site
+        @settings = settings
+        @index = LlmsDocumentIndex.new(settings, documents)
+        @context = Liquid::Context.new({}, {}, { site: site })
+      end
+
+      def to_s
+        return legacy_index if index.default_compatibility?
+
+        sections = [LlmsHeadings.new(site, settings).to_s(include_articles: false)] +
+                   index.sections.map { |section| render_section(section) }
+        "#{sections.reject(&:empty?).join("\n\n")}\n"
+      end
+
+      private
+
+      attr_reader :site, :settings, :index
+
+      def legacy_index
+        headings = LlmsHeadings.new(site, settings).to_s
+        links = sorted(index.sections.first&.documents || []).map { |document| entry(document) }.join("\n")
+        "#{[headings, links].reject(&:empty?).join("\n\n")}\n"
+      end
+
+      def render_section(section)
+        entries = sorted(section.documents).map { |document| entry(document) }
+        (["## #{LlmsText.literal(section.name)}"] + entries).join("\n")
+      end
+
+      def sorted(documents)
+        ordering = LlmsDocumentOrdering.new(settings)
+        dates = proc { |document| published_date(document) }
+        return ordering.legacy_ordered(documents, &dates) if index.default_compatibility?
+
+        ordering.ordered(documents, &dates)
+      end
+
+      def entry(document)
+        link = "- [#{title(document)}](#{escaped_link_url(absolute_markdown_url(document))})"
+        description = description_for(document)
+        link += ": #{description}" unless description.empty?
+        return link unless Configuration.enabled?(settings, "include_dates")
+
+        [link, DateMetadata.new(document.source_document.data).to_s].reject(&:empty?).join(" | ")
+      end
+
+      def absolute_markdown_url(document)
+        "#{site.config.fetch("url").sub(%r{/+\z}, "")}#{relative_url(document.markdown_url)}"
+      end
+
+      def title(document) = LlmsText.link_title(document.source_document)
+
+      def published_date(document) = DateMetadata.new(document.source_document.data).published_date
+
+      def description_for(document)
+        return "" unless Configuration.enabled?(settings, "include_descriptions")
+
+        description = description_value(document)
+        return "" if description.empty?
+
+        LlmsText.literal(inline_text(CGI.unescapeHTML(strip_html(rendered_description(description)))))
+      end
+
+      def description_value(document)
+        source = document.source_document
+        value = source.data["description"] || source.data["excerpt"]
+        value = source.excerpt if value.nil? && source.respond_to?(:excerpt)
+        value.to_s.strip
+      end
+
+      def rendered_description(description)
+        markdown_converter.convert(CGI.unescapeHTML(description))
+      end
+
+      def inline_text(value) = value.gsub(/\s+/, " ").strip
+
+      def markdown_converter
+        @markdown_converter ||= site.find_converter_instance(Jekyll::Converters::Markdown)
+      end
+
+      def escaped_link_url(url) = LlmsText.escaped_link_url(url)
+    end
+  end
+end

data/lib/jekyll/agent_markdown/llms_text.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Jekyll
+  module AgentMarkdown
+    module LlmsText
+      MARKDOWN_CHARACTERS = /[\\\[\]()*_~`<>!#|]/
+
+      module_function
+
+      def one_line(value) = value ? value.to_s.gsub(/\s+/, " ").strip : ""
+
+      def literal(value)
+        one_line(value).gsub("&", "&amp;").gsub(MARKDOWN_CHARACTERS) { |character| "\\#{character}" }
+      end
+
+      def link_title(source_document)
+        title = source_document.data.fetch("title", basename(source_document))
+        one_line(title.to_s).gsub(/[\\\[\]]/) { |character| "\\#{character}" }
+      end
+
+      def basename(source_document)
+        return source_document.basename_without_ext if source_document.respond_to?(:basename_without_ext)
+
+        source_document.basename
+      end
+
+      def escaped_link_url(url) = url.gsub("(", "%28").gsub(")", "%29")
+    end
+  end
+end

data/lib/jekyll/agent_markdown/metadata_footer.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Jekyll
+  module AgentMarkdown
+    class MetadataFooter
+      def initialize(entries)
+        @text = entries.compact.reject(&:empty?).join(" | ")
+      end
+
+      def append_to(content)
+        return content if text.empty?
+        return "#{text}\n" if content.empty?
+
+        "#{content}#{separator_for(content)}---\n#{text}\n"
+      end
+
+      private
+
+      attr_reader :text
+
+      def separator_for(content)
+        content.end_with?("\n") ? "\n" : "\n\n"
+      end
+    end
+  end
+end

data/lib/jekyll/agent_markdown/source_documents.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "jekyll"
+require_relative "configuration"
+
+module Jekyll
+  module AgentMarkdown
+    class SourceDocuments
+      include Enumerable
+
+      def initialize(site, settings, collection_validator:)
+        @site = site
+        @settings = settings
+        @collection_validator = collection_validator
+      end
+
+      def each
+        return enum_for(__method__) unless block_given?
+
+        posts.each { |document| yield document, :post, nil }
+        pages.each { |document| yield document, :page, nil }
+        collections.each { |document, collection_name| yield document, :collection, collection_name }
+      end
+
+      private
+
+      attr_reader :site, :settings, :collection_validator
+
+      def posts
+        return [] unless Configuration.enabled?(settings, "posts")
+
+        site.posts.docs
+      end
+
+      def pages
+        return [] unless Configuration.enabled?(settings, "pages")
+
+        site.pages.select { |page| authored_markdown_page?(page) }
+      end
+
+      def collections
+        Configuration.collection_names(settings).flat_map do |collection_name|
+          collection_validator.collection!(collection_name).docs.filter_map do |document|
+            [document, collection_name] if markdown_document?(document) && document.write?
+          end
+        end
+      end
+
+      def authored_markdown_page?(page)
+        page.is_a?(Jekyll::Page) &&
+          File.file?(site.in_source_dir(page.relative_path)) &&
+          markdown_extension?(page.ext)
+      end
+
+      def markdown_document?(document)
+        document.is_a?(Jekyll::Document) && markdown_extension?(document.extname)
+      end
+
+      def markdown_extension?(extension)
+        site.config.fetch("markdown_ext").split(",").any? do |markdown_extension|
+          ".#{markdown_extension.strip.downcase}" == extension.to_s.downcase
+        end
+      end
+    end
+  end
+end

data/lib/jekyll/agent_markdown/version.rb

@@ -2,6 +2,6 @@
 
 module Jekyll
   module AgentMarkdown
-    VERSION = "0.3.0"
+    VERSION = "0.4.0"
   end
 end

data/lib/jekyll-agent-markdown.rb

@@ -3,3 +3,6 @@
 require "jekyll"
 require_relative "jekyll/agent_markdown/version"
 require_relative "jekyll/agent_markdown/generator"
+require_relative "jekyll/agent_markdown/agent_markdown_link_tag"
+
+Liquid::Template.register_tag("agent_markdown_link", Jekyll::AgentMarkdown::AgentMarkdownLinkTag)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-agent-markdown
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Lucian Ghinda
@@ -29,8 +29,8 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '5'
-description: A Jekyll plugin that makes post Markdown available alongside rendered
-  HTML.
+description: A Jekyll plugin that exports posts, pages, and output collections as
+  Markdown siblings plus curated llms.txt and llms-full.txt files.
 email:
 - lucian@ghinda.com
 executables: []
@@ -40,14 +40,35 @@ files:
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
+- docs/deployment.md
+- examples/cloudflare/src/worker.js
+- examples/cloudflare/wrangler.toml
+- examples/netlify/netlify.toml
+- examples/netlify/netlify/edge-functions/markdown-negotiation.ts
+- examples/nginx/negotiation.js
+- examples/nginx/nginx.conf
 - lib/jekyll-agent-markdown.rb
+- lib/jekyll/agent_markdown/agent_markdown_link_tag.rb
+- lib/jekyll/agent_markdown/author_metadata.rb
+- lib/jekyll/agent_markdown/collection_validator.rb
 - lib/jekyll/agent_markdown/configuration.rb
 - lib/jekyll/agent_markdown/date_metadata.rb
 - lib/jekyll/agent_markdown/destination_claims.rb
+- lib/jekyll/agent_markdown/document_exporter.rb
+- lib/jekyll/agent_markdown/document_header.rb
+- lib/jekyll/agent_markdown/document_settings.rb
+- lib/jekyll/agent_markdown/exported_document.rb
 - lib/jekyll/agent_markdown/generator.rb
+- lib/jekyll/agent_markdown/llms_document_index.rb
+- lib/jekyll/agent_markdown/llms_document_ordering.rb
+- lib/jekyll/agent_markdown/llms_full_renderer.rb
 - lib/jekyll/agent_markdown/llms_headings.rb
+- lib/jekyll/agent_markdown/llms_index_renderer.rb
+- lib/jekyll/agent_markdown/llms_text.rb
 - lib/jekyll/agent_markdown/markdown_sibling_path.rb
+- lib/jekyll/agent_markdown/metadata_footer.rb
 - lib/jekyll/agent_markdown/raw_markdown_file.rb
+- lib/jekyll/agent_markdown/source_documents.rb
 - lib/jekyll/agent_markdown/version.rb
 homepage: https://github.com/lucianghinda/jekyll-agent-markdown
 licenses:
@@ -73,5 +94,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 4.0.11
 specification_version: 4
-summary: Exports Jekyll posts as raw Markdown siblings and llms.txt
+summary: Exports Jekyll content as Markdown siblings and llms.txt
 test_files: []