jekyll-ai-visible-content 0.1.0

data/lib/jekyll_ai_visible_content/generators/entity_map_generator.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module JekyllAiVisibleContent
+  module Generators
+    class EntityMapGenerator < Jekyll::Generator
+      safe true
+      priority :lowest
+
+      def generate(site)
+        config = JekyllAiVisibleContent.config(site)
+        return unless config.enabled?
+
+        registry = Entity::Registry.new(config)
+        scan_content(site, registry, config)
+
+        content = build_entity_map(config, registry)
+        page = Jekyll::PageWithoutAFile.new(site, site.source, '', 'entity-map.json')
+        page.content = content
+        page.data['layout'] = nil
+        page.data['sitemap'] = false
+        site.pages << page
+      end
+
+      private
+
+      def scan_content(site, registry, config)
+        all_docs = site.posts.docs + site.pages
+        entity_names = (config.entity['knows_about'] || []) + [config.entity['name']].compact
+
+        all_docs.each do |doc|
+          text = (doc.content || '').downcase
+          page_url = doc.url
+
+          entity_names.each do |name|
+            registry.record_mention(name, page_url) if text.include?(name.downcase)
+          end
+        end
+      end
+
+      def build_entity_map(config, registry)
+        primary = {
+          'id' => config.entity_id,
+          'type' => config.entity_type,
+          'name' => config.entity['name'],
+          'source_pages' => registry.pages_for(config.entity['name'] || ''),
+          'total_references' => registry.mention_count(config.entity['name'] || '')
+        }
+
+        topics = (config.entity['knows_about'] || []).map do |topic|
+          {
+            'name' => topic,
+            'mentions' => registry.mention_count(topic),
+            'linked_posts' => registry.pages_for(topic)
+          }
+        end
+
+        JSON.pretty_generate(
+          'primary_entity' => primary,
+          'entities' => topics,
+          'generated_at' => Time.now.iso8601
+        )
+      end
+    end
+  end
+end

data/lib/jekyll_ai_visible_content/generators/llms_txt_generator.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+module JekyllAiVisibleContent
+  module Generators
+    class LlmsTxtGenerator < Jekyll::Generator
+      safe true
+      priority :low
+
+      def generate(site)
+        config = JekyllAiVisibleContent.config(site)
+        return unless config.enabled? && config.llms_txt['enabled']
+
+        registry = Entity::Registry.new(config)
+
+        site.pages << build_llms_txt(site, config, registry)
+        site.pages << build_llms_full_txt(site, config, registry) if config.llms_txt['include_full_text']
+      end
+
+      private
+
+      def build_llms_txt(site, config, registry)
+        content = render_llms_txt(config, registry, site, full: false)
+        make_page(site, 'llms.txt', content)
+      end
+
+      def build_llms_full_txt(site, config, registry)
+        content = render_llms_txt(config, registry, site, full: true)
+        make_page(site, 'llms-full.txt', content)
+      end
+
+      def render_llms_txt(config, registry, site, full:)
+        lines = []
+        title = config.llms_txt['title'] || config.site_title
+        description = config.llms_txt['description'] || config.site_description
+
+        lines << "# #{title}"
+        lines << ''
+        lines << "> #{description.strip}" if description && !description.strip.empty?
+        lines << ''
+
+        append_entity_section(lines, config, registry)
+        append_topics_section(lines, config)
+        append_custom_sections(lines, config)
+        append_posts_section(lines, config, site, full: full)
+        append_links_section(lines, config)
+
+        lines.join("\n")
+      end
+
+      def append_entity_section(lines, config, _registry)
+        entity = config.entity
+        return unless entity['name']
+
+        lines << '## About'
+        lines << ''
+        lines << "#{entity['name']} is #{entity['description']&.strip}" if entity['description']
+        lines << ''
+
+        lines << "- Role: #{entity['job_title']}" if entity['job_title']
+        loc = entity['location']
+        lines << "- Location: #{[loc['locality'], loc['country']].compact.join(', ')}" if loc&.values&.any?
+        lines << ''
+      end
+
+      def append_topics_section(lines, config)
+        topics = config.entity['knows_about']
+        return unless topics&.any?
+
+        lines << '## Key Topics'
+        lines << ''
+        topics.each { |t| lines << "- #{t}" }
+        lines << ''
+      end
+
+      def append_custom_sections(lines, config)
+        sections = config.llms_txt['sections'] || []
+        sections.each do |section|
+          next unless section['heading']
+
+          lines << "## #{section['heading']}"
+          lines << ''
+          lines << section['content'].to_s.strip if section['content']
+          lines << ''
+        end
+      end
+
+      def append_posts_section(lines, config, site, full:)
+        posts = sorted_posts(site)
+        return if posts.empty?
+
+        lines << '## Posts'
+        lines << ''
+
+        posts.each do |post|
+          url = "#{config.site_url}#{post.url}"
+          desc = post.data['description']&.to_s&.strip
+
+          if full
+            lines << "### #{post.data['title']}"
+            lines << ''
+            lines << "URL: #{url}"
+            lines << "Date: #{post.data['date']&.strftime('%Y-%m-%d')}" if post.data['date']
+            lines << ''
+            lines << strip_html_and_liquid(post.content) if post.content
+            lines << ''
+            lines << '---'
+            lines << ''
+          else
+            entry = "- [#{post.data['title']}](#{url})"
+            entry += ": #{desc}" if desc && !desc.empty?
+            lines << entry
+          end
+        end
+
+        lines << ''
+      end
+
+      def append_links_section(lines, config)
+        lines << '## Links'
+        lines << ''
+        lines << "- About: #{config.site_url}/about/"
+
+        (config.entity['same_as'] || []).each do |link|
+          platform = extract_platform(link)
+          lines << "- #{platform}: #{link}"
+        end
+
+        lines << ''
+      end
+
+      def sorted_posts(site)
+        site.posts.docs.sort_by { |p| p.data['date'] || Time.at(0) }.reverse
+      end
+
+      def strip_html_and_liquid(text)
+        text.to_s
+            .gsub(/\{%.*?%\}/m, '')
+            .gsub(/\{\{.*?\}\}/m, '')
+            .gsub(/<[^>]+>/, '')
+            .gsub(/\n{3,}/, "\n\n")
+            .strip
+      end
+
+      def extract_platform(url)
+        case url
+        when /linkedin/i then 'LinkedIn'
+        when /github/i then 'GitHub'
+        when /twitter|x\.com/i then 'Twitter'
+        when /mastodon/i then 'Mastodon'
+        when /youtube/i then 'YouTube'
+        else
+          host = URI.parse(url).host
+          parts = host&.split('.')
+          name = parts && parts.length >= 2 ? parts[-2] : nil
+          name&.capitalize || 'Link'
+        end
+      rescue URI::InvalidURIError
+        'Link'
+      end
+
+      def make_page(site, name, content)
+        page = Jekyll::PageWithoutAFile.new(site, site.source, '', name)
+        page.content = content
+        page.data['layout'] = nil
+        page.data['sitemap'] = false
+        page
+      end
+    end
+  end
+end

data/lib/jekyll_ai_visible_content/generators/robots_txt_generator.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module JekyllAiVisibleContent
+  module Generators
+    class RobotsTxtGenerator < Jekyll::Generator
+      safe true
+      priority :low
+
+      CRAWLER_MAP = {
+        'allow_gptbot' => 'GPTBot',
+        'allow_perplexitybot' => 'PerplexityBot',
+        'allow_claudebot' => 'ClaudeBot',
+        'allow_googlebot' => 'Googlebot',
+        'allow_bingbot' => 'Bingbot'
+      }.freeze
+
+      def generate(site)
+        config = JekyllAiVisibleContent.config(site)
+        return unless config.enabled? && config.crawlers['generate_robots_txt']
+
+        content = render_robots_txt(config)
+        page = Jekyll::PageWithoutAFile.new(site, site.source, '', 'robots.txt')
+        page.content = content
+        page.data['layout'] = nil
+        page.data['sitemap'] = false
+        site.pages << page
+      end
+
+      private
+
+      def render_robots_txt(config)
+        lines = []
+        lines << 'User-agent: *'
+        lines << 'Allow: /'
+        lines << ''
+
+        CRAWLER_MAP.each do |key, agent|
+          next unless config.crawlers[key]
+
+          lines << "User-agent: #{agent}"
+          lines << 'Allow: /'
+          lines << ''
+        end
+
+        (config.crawlers['custom_rules'] || []).each do |rule|
+          lines << "User-agent: #{rule['user_agent']}"
+          lines << "#{rule['directive']}: #{rule['path']}" if rule['directive'] && rule['path']
+          lines << ''
+        end
+
+        lines << "Sitemap: #{config.site_url}/sitemap.xml"
+        lines << ''
+        lines.join("\n")
+      end
+    end
+  end
+end

data/lib/jekyll_ai_visible_content/hooks/post_render_hook.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module JekyllAiVisibleContent
+  module Hooks
+    module PostRenderHook
+      class << self
+        def register!
+          Jekyll::Hooks.register(:pages, :post_render) { |page| process(page) }
+          Jekyll::Hooks.register(:documents, :post_render) { |doc| process(doc) }
+        end
+
+        private
+
+        def process(doc)
+          return unless doc.output_ext == '.html'
+          return unless doc.output
+
+          site = doc.site
+          config = JekyllAiVisibleContent.config(site)
+          return unless config.enabled?
+
+          inject_json_ld(doc, config) if config.json_ld['auto_inject']
+          auto_link_entities(doc, config) if config.linking['enable_entity_links']
+        end
+
+        def inject_json_ld(doc, config)
+          return if doc.output.include?('application/ld+json')
+
+          registry = Entity::Registry.new(config)
+          builder = JsonLd::Builder.new(config, registry)
+
+          nodes = builder.build_for_page(doc)
+          return if nodes.empty?
+
+          if config.seo_tag_present?
+            nodes.reject! { |n| n['@type'] == 'WebSite' }
+            return if nodes.empty?
+          end
+
+          script_tag = builder.to_script_tag(nodes)
+          doc.output = doc.output.sub('</head>', "#{script_tag}\n</head>")
+        end
+
+        def auto_link_entities(doc, config)
+          return unless doc.output
+
+          registry = Entity::Registry.new(config)
+          definitions = registry.entity_definitions
+          max_per = config.linking['max_links_per_entity_per_post'] || 1
+
+          definitions.each_value do |defn|
+            name = defn['name']
+            url = defn['url']
+            next unless name && url
+
+            link_html = %(<a href="#{url}" itemprop="about" itemscope ) +
+                        %(itemtype="https://schema.org/Thing"><span itemprop="name">#{name}</span></a>)
+
+            count = 0
+            doc.output = doc.output.gsub(/(?<=\s|>)#{Regexp.escape(name)}(?=[\s,.<])/i) do |match|
+              if count < max_per && !inside_tag?(doc.output, Regexp.last_match.begin(0))
+                count += 1
+                link_html
+              else
+                match
+              end
+            end
+          end
+        end
+
+        def inside_tag?(html, position)
+          preceding = html[0...position]
+          last_open = preceding.rindex('<') || -1
+          last_close = preceding.rindex('>') || -1
+          last_open > last_close
+        end
+      end
+    end
+  end
+end
+
+JekyllAiVisibleContent::Hooks::PostRenderHook.register!

data/lib/jekyll_ai_visible_content/hooks/validate_hook.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module JekyllAiVisibleContent
+  module Hooks
+    module ValidateHook
+      class << self
+        def register!
+          Jekyll::Hooks.register(:site, :post_write) { |site| validate(site) }
+        end
+
+        private
+
+        def validate(site)
+          config = JekyllAiVisibleContent.config(site)
+          return unless config.enabled?
+
+          warnings = []
+          errors = []
+
+          entity_validator = Validators::EntityConsistencyValidator.new(config, site)
+          warnings.concat(entity_validator.validate)
+
+          json_ld_validator = Validators::JsonLdValidator.new(config)
+          errors.concat(json_ld_validator.validate)
+
+          link_validator = Validators::LinkValidator.new(config, site)
+          warnings.concat(link_validator.validate)
+
+          print_results(warnings, errors)
+          abort_if_needed(errors, config) if config.validation['fail_build_on_error'] && errors.any?
+        end
+
+        def print_results(warnings, errors)
+          return if warnings.empty? && errors.empty?
+
+          Jekyll.logger.info 'AI Visible Content:', 'Validation report'
+          warnings.each { |w| Jekyll.logger.warn 'AI Visible Content:', w }
+          errors.each { |e| Jekyll.logger.error 'AI Visible Content:', e }
+        end
+
+        def abort_if_needed(errors, _config)
+          raise JekyllAiVisibleContent::Error, "Build failed: #{errors.size} validation error(s)" if errors.any?
+        end
+      end
+    end
+  end
+end
+
+JekyllAiVisibleContent::Hooks::ValidateHook.register!

data/lib/jekyll_ai_visible_content/json_ld/blog_posting_schema.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module JekyllAiVisibleContent
+  module JsonLd
+    class BlogPostingSchema
+      attr_reader :config, :registry, :page
+
+      def initialize(config, registry, page)
+        @config = config
+        @registry = registry
+        @page = page
+      end
+
+      def build
+        data = page.data
+        url = absolute_url(page.url)
+
+        posting = {
+          '@type' => 'BlogPosting',
+          '@id' => "#{url}#article",
+          'mainEntityOfPage' => {
+            '@type' => 'WebPage',
+            '@id' => url
+          },
+          'headline' => data['title'],
+          'description' => data['description']&.to_s&.strip,
+          'datePublished' => format_date(data['date']),
+          'dateModified' => format_date(data['last_modified_at'] || data['date']),
+          'author' => registry.primary_entity_ref,
+          'publisher' => registry.primary_entity_ref,
+          'isPartOf' => { '@id' => "#{config.site_url}/#website" }
+        }
+
+        append_image(posting, data)
+        append_keywords(posting, data)
+        append_about(posting, data)
+        append_word_count(posting)
+        append_article_body(posting)
+
+        posting.compact
+      end
+
+      private
+
+      def append_image(posting, data)
+        return unless data['image']
+
+        posting['image'] = absolute_url(data['image'])
+      end
+
+      def append_keywords(posting, data)
+        tags = data['tags'] || data['keywords']
+        posting['keywords'] = tags if tags&.any?
+      end
+
+      def append_about(posting, data)
+        topics = data['topics'] || data['categories']
+        return unless topics&.any?
+
+        posting['about'] = topics.map { |t| { '@type' => 'Thing', 'name' => t } }
+      end
+
+      def append_word_count(posting)
+        content = page.content
+        return unless content
+
+        posting['wordCount'] = content.split(/\s+/).size
+      end
+
+      def append_article_body(posting)
+        mode = config.json_ld['article_body']
+        case mode
+        when 'full'
+          posting['articleBody'] = strip_html(page.content) if page.content
+        when 'excerpt'
+          excerpt_text = page.data['description'] || page.data['excerpt']&.to_s
+          posting['articleBody'] = strip_html(excerpt_text.to_s).strip if excerpt_text
+        end
+      end
+
+      def format_date(date)
+        return nil unless date
+
+        if date.respond_to?(:iso8601)
+          date.iso8601
+        elsif date.respond_to?(:strftime)
+          date.strftime('%Y-%m-%dT%H:%M:%S%:z')
+        else
+          date.to_s
+        end
+      end
+
+      def absolute_url(path)
+        return path if path&.start_with?('http')
+
+        "#{config.site_url}#{path}"
+      end
+
+      def strip_html(text)
+        text.to_s.gsub(/<[^>]+>/, '').gsub(/\s+/, ' ').strip
+      end
+    end
+  end
+end

data/lib/jekyll_ai_visible_content/json_ld/breadcrumb_schema.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module JekyllAiVisibleContent
+  module JsonLd
+    class BreadcrumbSchema
+      attr_reader :config, :page
+
+      def initialize(config, page)
+        @config = config
+        @page = page
+      end
+
+      def build
+        items = build_items
+        return nil if items.size < 2
+
+        {
+          '@type' => 'BreadcrumbList',
+          'itemListElement' => items
+        }
+      end
+
+      private
+
+      def build_items
+        url = page.respond_to?(:url) ? page.url : '/'
+        segments = url.to_s.split('/').reject(&:empty?)
+
+        items = [list_item(1, 'Home', "#{config.site_url}/")]
+
+        path = ''
+        segments.each_with_index do |segment, idx|
+          path = "#{path}/#{segment}"
+          position = idx + 2
+          name = humanize(segment)
+
+          items << if idx == segments.size - 1
+                     list_item_no_url(position, page.data['title'] || name)
+                   else
+                     list_item(position, name, "#{config.site_url}#{path}/")
+                   end
+        end
+
+        items
+      end
+
+      def list_item(position, name, url)
+        {
+          '@type' => 'ListItem',
+          'position' => position,
+          'name' => name,
+          'item' => url
+        }
+      end
+
+      def list_item_no_url(position, name)
+        {
+          '@type' => 'ListItem',
+          'position' => position,
+          'name' => name
+        }
+      end
+
+      def humanize(slug)
+        slug.gsub(/[-_]/, ' ').gsub(/\b\w/, &:upcase)
+      end
+    end
+  end
+end

data/lib/jekyll_ai_visible_content/json_ld/builder.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module JekyllAiVisibleContent
+  module JsonLd
+    class Builder
+      attr_reader :config, :registry
+
+      def initialize(config, registry)
+        @config = config
+        @registry = registry
+      end
+
+      def build_for_page(page)
+        nodes = []
+        page_data = page.data
+
+        if person_page?(page)
+          nodes << PersonSchema.new(config, registry).build
+          nodes << WebsiteSchema.new(config, registry).build if config.json_ld['include_website_schema']
+        elsif post_page?(page) && config.json_ld['include_blog_posting']
+          nodes << BlogPostingSchema.new(config, registry, page).build
+        end
+
+        if config.json_ld['include_breadcrumbs']
+          crumbs = BreadcrumbSchema.new(config, page).build
+          nodes << crumbs if crumbs
+        end
+
+        nodes << FaqSchema.new(config, page).build if config.json_ld['include_faq'] && page_data['faq']&.any?
+
+        nodes << HowToSchema.new(config, page).build if config.json_ld['include_how_to'] && page_data['how_to']
+
+        nodes.compact
+      end
+
+      def build_for_homepage
+        nodes = []
+        nodes << PersonSchema.new(config, registry).build if registry.primary_entity
+        nodes << WebsiteSchema.new(config, registry).build if config.json_ld['include_website_schema']
+        nodes.compact
+      end
+
+      def to_script_tag(nodes, compact: nil)
+        return '' if nodes.empty?
+
+        use_compact = compact.nil? ? config.json_ld['compact'] : compact
+        graph = { '@context' => 'https://schema.org', '@graph' => nodes }
+        json = use_compact ? JSON.generate(graph) : JSON.pretty_generate(graph)
+        %(<script type="application/ld+json">\n#{json}\n</script>)
+      end
+
+      private
+
+      def person_page?(page)
+        page.data['entity_type'] == 'Person' ||
+          (page.respond_to?(:url) && page.url&.match?(%r{/about/?$}))
+      end
+
+      def post_page?(page)
+        page.respond_to?(:collection) && page.collection&.label == 'posts'
+      end
+    end
+  end
+end