jekyll-ai-visible-content 0.1.0

data/jekyll-ai-visible-content.gemspec

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative 'lib/jekyll_ai_visible_content/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'jekyll-ai-visible-content'
+  spec.version       = JekyllAiVisibleContent::VERSION
+  spec.authors       = ['madmatvey']
+  spec.email         = ['potehin@gmail.com']
+
+  spec.summary       = 'Jekyll plugin that maximizes AI search discoverability'
+  spec.description   = 'Adds rich JSON-LD structured data, llms.txt, semantic HTML helpers, ' \
+                       'entity identity management, and AI crawler policies to Jekyll sites.'
+  spec.homepage      = 'https://github.com/madmatvey/jekyll-ai-visible-content'
+  spec.license       = 'MIT'
+
+  spec.required_ruby_version = '>= 3.2'
+  spec.metadata['homepage_uri']    = spec.homepage
+  spec.metadata['source_code_uri'] = spec.homepage
+  spec.metadata['changelog_uri']   = "#{spec.homepage}/blob/master/CHANGELOG.md"
+  spec.metadata['rubygems_mfa_required'] = 'true'
+
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:spec|\.github)/}) }
+  end
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'jekyll', '>= 4.0', '< 5.0'
+end

data/lib/jekyll-ai-visible-content.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'jekyll'
+require 'json'
+
+require_relative 'jekyll_ai_visible_content/version'
+require_relative 'jekyll_ai_visible_content/configuration'
+require_relative 'jekyll_ai_visible_content/entity/person'
+require_relative 'jekyll_ai_visible_content/entity/organization'
+require_relative 'jekyll_ai_visible_content/entity/registry'
+require_relative 'jekyll_ai_visible_content/json_ld/builder'
+require_relative 'jekyll_ai_visible_content/json_ld/person_schema'
+require_relative 'jekyll_ai_visible_content/json_ld/blog_posting_schema'
+require_relative 'jekyll_ai_visible_content/json_ld/website_schema'
+require_relative 'jekyll_ai_visible_content/json_ld/breadcrumb_schema'
+require_relative 'jekyll_ai_visible_content/json_ld/faq_schema'
+require_relative 'jekyll_ai_visible_content/json_ld/how_to_schema'
+require_relative 'jekyll_ai_visible_content/json_ld/collection_schema'
+require_relative 'jekyll_ai_visible_content/generators/llms_txt_generator'
+require_relative 'jekyll_ai_visible_content/generators/robots_txt_generator'
+require_relative 'jekyll_ai_visible_content/generators/entity_map_generator'
+require_relative 'jekyll_ai_visible_content/generators/content_graph_generator'
+require_relative 'jekyll_ai_visible_content/tags/ai_json_ld_tag'
+require_relative 'jekyll_ai_visible_content/tags/ai_author_tag'
+require_relative 'jekyll_ai_visible_content/tags/ai_entity_link_tag'
+require_relative 'jekyll_ai_visible_content/tags/ai_related_posts_tag'
+require_relative 'jekyll_ai_visible_content/tags/ai_breadcrumb_tag'
+require_relative 'jekyll_ai_visible_content/filters/naming_filter'
+require_relative 'jekyll_ai_visible_content/filters/entity_filter'
+require_relative 'jekyll_ai_visible_content/hooks/post_render_hook'
+require_relative 'jekyll_ai_visible_content/hooks/validate_hook'
+require_relative 'jekyll_ai_visible_content/validators/entity_consistency_validator'
+require_relative 'jekyll_ai_visible_content/validators/json_ld_validator'
+require_relative 'jekyll_ai_visible_content/validators/link_validator'
+
+module JekyllAiVisibleContent
+  class Error < StandardError; end
+
+  def self.config(site)
+    @configs ||= {}.compare_by_identity
+    @configs[site] ||= Configuration.new(site)
+  end
+
+  def self.reset!
+    @configs = {}.compare_by_identity
+  end
+end

data/lib/jekyll_ai_visible_content/configuration.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module JekyllAiVisibleContent
+  class Configuration
+    CONFIG_KEY = 'ai_visible_content'
+
+    DEFAULTS = {
+      'enabled' => true,
+      'entity' => {
+        'type' => 'Person',
+        'id_slug' => nil,
+        'name' => nil,
+        'alternate_names' => [],
+        'job_title' => nil,
+        'description' => nil,
+        'image' => nil,
+        'email' => nil,
+        'location' => { 'locality' => nil, 'country' => nil },
+        'knows_about' => [],
+        'same_as' => [],
+        'works_for' => nil,
+        'occupation' => nil
+      },
+      'json_ld' => {
+        'auto_inject' => true,
+        'include_website_schema' => true,
+        'include_breadcrumbs' => true,
+        'include_blog_posting' => true,
+        'include_faq' => true,
+        'include_how_to' => true,
+        'article_body' => 'excerpt',
+        'compact' => false
+      },
+      'crawlers' => {
+        'allow_gptbot' => true,
+        'allow_perplexitybot' => true,
+        'allow_claudebot' => true,
+        'allow_googlebot' => true,
+        'allow_bingbot' => true,
+        'custom_rules' => [],
+        'generate_robots_txt' => true
+      },
+      'llms_txt' => {
+        'enabled' => true,
+        'title' => nil,
+        'description' => nil,
+        'sections' => [],
+        'include_full_text' => true
+      },
+      'linking' => {
+        'enable_entity_links' => true,
+        'entity_definitions' => {},
+        'max_links_per_entity_per_post' => 1,
+        'enable_related_posts' => true,
+        'related_posts_limit' => 3
+      },
+      'validation' => {
+        'warn_name_inconsistency' => true,
+        'warn_missing_same_as' => true,
+        'warn_missing_dates' => true,
+        'warn_orphan_pages' => true,
+        'warn_missing_descriptions' => true,
+        'fail_build_on_error' => false
+      }
+    }.freeze
+
+    attr_reader :site
+
+    def initialize(site)
+      @site = site
+      @raw = deep_merge(DEFAULTS, site.config.fetch(CONFIG_KEY, {}))
+    end
+
+    def enabled?
+      @raw['enabled'] == true
+    end
+
+    def entity
+      @raw['entity']
+    end
+
+    def entity_id
+      slug = entity['id_slug'] || normalize_slug(entity['name'])
+      "#{site_url}/##{slug}" if slug
+    end
+
+    def entity_type
+      entity['type'] || 'Person'
+    end
+
+    def json_ld
+      @raw['json_ld']
+    end
+
+    def crawlers
+      @raw['crawlers']
+    end
+
+    def llms_txt
+      @raw['llms_txt']
+    end
+
+    def linking
+      @raw['linking']
+    end
+
+    def validation
+      @raw['validation']
+    end
+
+    def site_url
+      @site.config['url'] || ''
+    end
+
+    def site_title
+      @site.config['title'] || entity['name'] || ''
+    end
+
+    def site_description
+      @site.config['description'] || entity['description'] || ''
+    end
+
+    def seo_tag_present?
+      @site.config['plugins']&.include?('jekyll-seo-tag') ||
+        Gem.loaded_specs.key?('jekyll-seo-tag')
+    rescue StandardError
+      false
+    end
+
+    def [](key)
+      @raw[key]
+    end
+
+    private
+
+    def normalize_slug(value)
+      return nil unless value
+
+      value.downcase.gsub(/[^a-z0-9]+/, '-').gsub(/(^-|-$)/, '')
+    end
+
+    def deep_merge(base, override)
+      base.each_with_object(base.dup) do |(key, base_val), result|
+        next unless override.key?(key)
+
+        result[key] = if base_val.is_a?(Hash) && override[key].is_a?(Hash)
+                        deep_merge(base_val, override[key])
+                      else
+                        override[key]
+                      end
+      end.merge(override.reject { |k, _| base.key?(k) })
+    end
+  end
+end

data/lib/jekyll_ai_visible_content/entity/organization.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module JekyllAiVisibleContent
+  module Entity
+    class Organization
+      attr_reader :config
+
+      def initialize(config)
+        @config = config
+      end
+
+      def to_hash
+        entity = config.entity
+        data = {
+          '@type' => 'Organization',
+          '@id' => config.entity_id,
+          'name' => entity['name'],
+          'url' => config.site_url
+        }
+
+        append_image(data, entity)
+        append_description(data, entity)
+        append_address(data, entity)
+        append_same_as(data, entity)
+
+        data.compact
+      end
+
+      private
+
+      def append_image(data, entity)
+        return unless entity['image']
+
+        url = absolute_url(entity['image'])
+        data['logo'] = {
+          '@type' => 'ImageObject',
+          'url' => url
+        }
+      end
+
+      def append_description(data, entity)
+        data['description'] = entity['description']&.strip if entity['description']
+      end
+
+      def append_address(data, entity)
+        loc = entity['location']
+        return unless loc && (loc['locality'] || loc['country'])
+
+        data['address'] = {
+          '@type' => 'PostalAddress',
+          'addressLocality' => loc['locality'],
+          'addressCountry' => loc['country']
+        }.compact
+      end
+
+      def append_same_as(data, entity)
+        links = entity['same_as']
+        data['sameAs'] = links if links&.any?
+      end
+
+      def absolute_url(path)
+        return path if path&.start_with?('http')
+
+        "#{config.site_url}#{path}"
+      end
+    end
+  end
+end

data/lib/jekyll_ai_visible_content/entity/person.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module JekyllAiVisibleContent
+  module Entity
+    class Person
+      attr_reader :config
+
+      def initialize(config)
+        @config = config
+      end
+
+      def to_hash
+        entity = config.entity
+        data = {
+          '@type' => 'Person',
+          '@id' => config.entity_id,
+          'name' => entity['name'],
+          'url' => config.site_url
+        }
+
+        append_alternate_names(data, entity)
+        append_image(data, entity)
+        append_simple_fields(data, entity)
+        append_address(data, entity)
+        append_knows_about(data, entity)
+        append_same_as(data, entity)
+        append_works_for(data, entity)
+        append_occupation(data, entity)
+
+        data.compact
+      end
+
+      private
+
+      def append_alternate_names(data, entity)
+        names = entity['alternate_names']
+        data['alternateName'] = names if names&.any?
+      end
+
+      def append_image(data, entity)
+        return unless entity['image']
+
+        url = absolute_url(entity['image'])
+        data['image'] = {
+          '@type' => 'ImageObject',
+          'url' => url
+        }
+      end
+
+      def append_simple_fields(data, entity)
+        data['jobTitle'] = entity['job_title'] if entity['job_title']
+        data['description'] = entity['description']&.strip if entity['description']
+        data['email'] = entity['email'] if entity['email']
+      end
+
+      def append_address(data, entity)
+        loc = entity['location']
+        return unless loc && (loc['locality'] || loc['country'])
+
+        data['address'] = {
+          '@type' => 'PostalAddress',
+          'addressLocality' => loc['locality'],
+          'addressCountry' => loc['country']
+        }.compact
+      end
+
+      def append_knows_about(data, entity)
+        items = entity['knows_about']
+        data['knowsAbout'] = items if items&.any?
+      end
+
+      def append_same_as(data, entity)
+        links = entity['same_as']
+        data['sameAs'] = links if links&.any?
+      end
+
+      def append_works_for(data, entity)
+        wf = entity['works_for']
+        return unless wf
+
+        data['worksFor'] = {
+          '@type' => wf['type'] || 'Organization',
+          'name' => wf['name']
+        }.compact
+      end
+
+      def append_occupation(data, entity)
+        occ = entity['occupation']
+        return unless occ
+
+        occupation = {
+          '@type' => 'Occupation',
+          'name' => occ['name']
+        }
+
+        if occ['location_country']
+          occupation['occupationLocation'] = {
+            '@type' => 'Country',
+            'name' => occ['location_country']
+          }
+        end
+
+        occupation['skills'] = occ['skills'] if occ['skills']
+        data['hasOccupation'] = occupation.compact
+      end
+
+      def absolute_url(path)
+        return path if path&.start_with?('http')
+
+        "#{config.site_url}#{path}"
+      end
+    end
+  end
+end

data/lib/jekyll_ai_visible_content/entity/registry.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module JekyllAiVisibleContent
+  module Entity
+    class Registry
+      attr_reader :config, :primary_entity, :topic_entities
+
+      def initialize(config)
+        @config = config
+        @primary_entity = build_primary_entity
+        @topic_entities = build_topic_entities
+        @mention_counts = Hash.new(0)
+        @entity_pages = Hash.new { |h, k| h[k] = [] }
+      end
+
+      def primary_entity_hash
+        @primary_entity_hash ||= primary_entity&.to_hash
+      end
+
+      def primary_entity_ref
+        return nil unless config.entity_id
+
+        { '@type' => config.entity_type, '@id' => config.entity_id }
+      end
+
+      def record_mention(entity_name, page_url)
+        normalized = normalize_name(entity_name)
+        @mention_counts[normalized] += 1
+        @entity_pages[normalized] << page_url unless @entity_pages[normalized].include?(page_url)
+      end
+
+      def mention_count(entity_name)
+        @mention_counts[normalize_name(entity_name)]
+      end
+
+      def pages_for(entity_name)
+        @entity_pages[normalize_name(entity_name)]
+      end
+
+      def all_mentions
+        @mention_counts.dup
+      end
+
+      def topic_url(topic_name)
+        slug = normalize_name(topic_name).gsub(/\s+/, '-')
+        "/topics/#{slug}/"
+      end
+
+      def entity_definitions
+        defs = config.linking['entity_definitions'] || {}
+        knows = config.entity['knows_about'] || []
+
+        knows.each_with_object(defs.dup) do |topic, result|
+          slug = normalize_name(topic).gsub(/\s+/, '-')
+          next if result.key?(slug)
+
+          result[slug] = {
+            'name' => topic,
+            'url' => topic_url(topic),
+            'description' => nil
+          }
+        end
+      end
+
+      def find_entity_by_name(name)
+        normalized = normalize_name(name)
+        entity_definitions.values.find { |d| normalize_name(d['name']) == normalized }
+      end
+
+      private
+
+      def build_primary_entity
+        return nil unless config.entity['name']
+
+        case config.entity_type
+        when 'Person'
+          Person.new(config)
+        when 'Organization'
+          Organization.new(config)
+        end
+      end
+
+      def build_topic_entities
+        (config.entity['knows_about'] || []).map do |topic|
+          { 'name' => topic, 'slug' => normalize_name(topic).gsub(/\s+/, '-') }
+        end
+      end
+
+      def normalize_name(name)
+        name.to_s.strip.downcase
+      end
+    end
+  end
+end

data/lib/jekyll_ai_visible_content/filters/entity_filter.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module JekyllAiVisibleContent
+  module Filters
+    module EntityFilter
+      def ai_entity_name(input)
+        site = @context.registers[:site]
+        config = JekyllAiVisibleContent.config(site)
+        config.entity['name'] || input
+      end
+
+      def ai_entity_id(input)
+        site = @context.registers[:site]
+        config = JekyllAiVisibleContent.config(site)
+        config.entity_id || input
+      end
+
+      def ai_entity_url(entity_name)
+        site = @context.registers[:site]
+        config = JekyllAiVisibleContent.config(site)
+        registry = Entity::Registry.new(config)
+        definition = registry.find_entity_by_name(entity_name)
+        definition ? definition['url'] : '#'
+      end
+    end
+  end
+end
+
+Liquid::Template.register_filter(JekyllAiVisibleContent::Filters::EntityFilter)

data/lib/jekyll_ai_visible_content/filters/naming_filter.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module JekyllAiVisibleContent
+  module Filters
+    module NamingFilter
+      def ai_slugify(input)
+        input.to_s.strip.downcase
+             .gsub(/[^a-z0-9\s-]/, '')
+             .gsub(/\s+/, '-')
+             .gsub(/-+/, '-')
+             .gsub(/(^-|-$)/, '')
+      end
+
+      def ai_entity_slug(input)
+        ai_slugify(input)
+      end
+
+      def ai_canonical_url(input)
+        url = input.to_s.strip
+        url = "#{url}/" unless url.end_with?('/') || url.match?(/\.\w+$/)
+        url
+      end
+    end
+  end
+end
+
+Liquid::Template.register_filter(JekyllAiVisibleContent::Filters::NamingFilter)

data/lib/jekyll_ai_visible_content/generators/content_graph_generator.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module JekyllAiVisibleContent
+  module Generators
+    class ContentGraphGenerator < Jekyll::Generator
+      safe true
+      priority :lowest
+
+      def generate(site)
+        config = JekyllAiVisibleContent.config(site)
+        return unless config.enabled?
+
+        graph = build_link_graph(site, config)
+        orphans = find_orphans(graph, site)
+
+        site.data['ai_content_graph'] = graph
+        site.data['ai_orphan_pages'] = orphans
+      end
+
+      private
+
+      def build_link_graph(site, config)
+        graph = Hash.new { |h, k| h[k] = { 'outbound' => [], 'inbound' => [] } }
+
+        all_docs(site).each do |doc|
+          source_url = doc.url
+          links = extract_internal_links(doc.content || '', config.site_url)
+
+          links.each do |target_url|
+            graph[source_url]['outbound'] << target_url unless graph[source_url]['outbound'].include?(target_url)
+            graph[target_url]['inbound'] << source_url unless graph[target_url]['inbound'].include?(source_url)
+          end
+        end
+
+        graph
+      end
+
+      def find_orphans(graph, site)
+        all_urls = all_docs(site).map(&:url)
+        all_urls.select { |url| (graph[url]['inbound'] || []).empty? && url != '/' }
+      end
+
+      def extract_internal_links(content, site_url)
+        links = []
+        content.scan(/href=["']([^"']+)["']/) do |match|
+          href = match[0]
+          next if href.start_with?('#', 'mailto:', 'tel:', 'javascript:')
+
+          if href.start_with?('/')
+            links << normalize_url(href)
+          elsif href.start_with?(site_url)
+            links << normalize_url(href.sub(site_url, ''))
+          end
+        end
+        links.uniq
+      end
+
+      def normalize_url(url)
+        path = url.split('?').first.split('#').first
+        path = "#{path}/" unless path.end_with?('/') || path.match?(/\.\w+$/)
+        path
+      end
+
+      def all_docs(site)
+        site.posts.docs + site.pages
+      end
+    end
+  end
+end