trifle-docs 0.5.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b52d0fa673ad7999994caf2fda33169ac1815587cbf3023cce7a3a5ec65373a2
-  data.tar.gz: 56c8c894a50aef901915dee9eab80a2f7b7e5a6fe3b46cbe344f4360f0c639b1
+  metadata.gz: 9f51e6c6b892eaca5b790d652ccb970146c9e98b624cf3353c84af3ffb95ed4c
+  data.tar.gz: f5058c9fa39c81a635db554fabffaab5535f57a76a078138319df0e72732cd49
 SHA512:
-  metadata.gz: 59e6d106be4c79000a4470429308ce833ba8a6e0e3d02d8b1bdfacafd638030e14687b53aa5f5d81f55b91158b8873d9eb624bc0665d04d3bb781afc31927c8d
-  data.tar.gz: d7ca56cd6809bd75891a5bf6b8fd49847007539dced62b5c69b43d9c779e00d963c3986fba84ff457d68edd52947cc5a554333584325ac56bb42e2f95d4e5579
+  metadata.gz: 48ce9ec16b882be46f90825f0e12bb13c783fa16e5b22f5eae27551418a5e58d112a321496fdf46810474c68de5f8933661931a696b6378bc836ffd7a8e74192
+  data.tar.gz: 9721a213e1f2cef167e0adf4ed18025b0740ba6594e26f6f0f326e4e6412631f5957447f503cd2ca45c4ce4e2df5efb497a15a8aaf2943a29b246a2c373869a8

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    trifle-docs (0.5.0)
+    trifle-docs (0.7.0)
       redcarpet
       rouge
       sinatra
@@ -14,7 +14,8 @@ GEM
     base64 (0.3.0)
     byebug (11.1.3)
     diff-lcs (1.5.0)
-    mustermann (3.0.3)
+    logger (1.7.0)
+    mustermann (3.0.4)
       ruby2_keywords (~> 0.0.1)
     nio4r (2.7.4)
     parallel (1.22.1)
@@ -22,18 +23,20 @@ GEM
       ast (~> 2.4.1)
     puma (6.6.0)
       nio4r (~> 2.0)
-    rack (3.1.16)
-    rack-protection (4.0.0)
+    rack (3.2.4)
+    rack-protection (4.2.1)
       base64 (>= 0.1.0)
+      logger (>= 1.6.0)
       rack (>= 3.0.0, < 4)
-    rack-session (2.0.0)
+    rack-session (2.1.1)
+      base64 (>= 0.1.0)
       rack (>= 3.0.0)
     rainbow (3.1.1)
     rake (13.0.6)
-    redcarpet (3.6.0)
+    redcarpet (3.6.1)
     regexp_parser (2.5.0)
     rexml (3.2.5)
-    rouge (4.2.0)
+    rouge (4.7.0)
     rspec (3.11.0)
       rspec-core (~> 3.11.0)
       rspec-expectations (~> 3.11.0)
@@ -60,15 +63,16 @@ GEM
       parser (>= 3.1.1.0)
     ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.5)
-    sinatra (4.0.0)
+    sinatra (4.2.1)
+      logger (>= 1.6.0)
       mustermann (~> 3.0)
       rack (>= 3.0.0, < 4)
-      rack-protection (= 4.0.0)
+      rack-protection (= 4.2.1)
       rack-session (>= 2.0.0, < 3)
       tilt (~> 2.0)
-    tilt (2.3.0)
+    tilt (2.7.0)
     unicode-display_width (2.1.0)
-    yaml (0.3.0)
+    yaml (0.4.0)
 
 PLATFORMS
   arm64-darwin-21

data/lib/trifle/docs/app.rb

@@ -25,32 +25,92 @@ module Trifle
         )
       end
 
+      get '/llms.txt' do
+        meta = Trifle::Docs.meta(url: 'llms.txt')
+        return send_file(meta['path']) if meta && meta['type'] == 'file'
+
+        content = Trifle::Docs::Helper::Llms.homepage_markdown
+        halt(404, 'Not Found') if content.nil?
+
+        content_type 'text/markdown'
+        content
+      end
+
+      get '/llms-full.txt' do
+        meta = Trifle::Docs.meta(url: 'llms-full.txt')
+        return send_file(meta['path']) if meta && meta['type'] == 'file'
+
+        content = Trifle::Docs::Helper::Llms.full_markdown
+        halt(404, 'Not Found') if content.nil? || content.strip.empty?
+
+        content_type 'text/markdown'
+        content
+      end
+
+      get '/sitemap.xml' do
+        meta = Trifle::Docs.meta(url: 'sitemap.xml')
+        return send_file(meta['path']) if meta && meta['type'] == 'file'
+
+        render_generated_sitemap
+      end
+
       get '/*' do
+        handle_request(params, request)
+      end
+
+      def markdown_requested?(request, params)
+        return true if params['format'].to_s.downcase == 'md'
+
+        accept = request.env['HTTP_ACCEPT'].to_s
+        accept.include?('text/markdown')
+      end
+
+      def render_markdown?(meta, request, params)
+        return false if meta['type'] == 'file'
+
+        markdown_requested?(request, params) ||
+          Trifle::Docs::Helper::AiDetection.ai_scraper?(request.user_agent)
+      end
+
+      def handle_request(params, request)
         url = params['splat'].first.chomp('/')
         meta = Trifle::Docs.meta(url: url)
         halt(404, 'Not Found') if meta.nil?
 
-        if Trifle::Docs::Helper::AiDetection.ai_scraper?(request.user_agent) && meta['type'] != 'file'
-          content_type 'text/markdown'
-          return Trifle::Docs::Helper::MarkdownLayout.render(
-            meta: meta,
-            raw_content: Trifle::Docs.raw_content(url: url),
-            sitemap: Trifle::Docs.sitemap
-          )
-        end
-
-        if meta['type'] == 'file'
-          send_file meta['path']
-        else
-          erb (meta['template'] || 'page').to_sym, {}, {
-            sitemap: Trifle::Docs.sitemap,
-            collection: Trifle::Docs.collection(url: url),
-            content: Trifle::Docs.content(url: url),
-            meta: meta,
-            url: url
-          }
-        end
+        return render_markdown(meta, url) if render_markdown?(meta, request, params)
+        return send_file(meta['path']) if meta['type'] == 'file'
+
+        render_html(meta, url)
       end
+
+      def render_markdown(meta, url)
+        content_type 'text/markdown'
+        Trifle::Docs::Helper::MarkdownLayout.render(
+          meta: meta,
+          raw_content: Trifle::Docs.raw_content(url: url),
+          sitemap: Trifle::Docs.sitemap
+        )
+      end
+
+      def render_html(meta, url)
+        erb (meta['template'] || 'page').to_sym, {}, {
+          sitemap: Trifle::Docs.sitemap,
+          collection: Trifle::Docs.collection(url: url),
+          content: Trifle::Docs.content(url: url),
+          meta: meta,
+          url: url
+        }
+      end
+
+      def render_generated_sitemap
+        content = Trifle::Docs::Helper::Sitemap.xml
+        halt(404, 'Not Found') if content.nil? || content.strip.empty?
+
+        content_type 'application/xml'
+        content
+      end
+
+      private :handle_request
     end
   end
 end

data/lib/trifle/docs/engine.rb

@@ -17,47 +17,37 @@ if Object.const_defined?('Rails')
         def self.draw
           Trifle::Docs::Engine.routes.draw do
             root to: 'page#show'
+            get 'llms.txt', to: 'page#llms'
+            get 'llms-full.txt', to: 'page#llms_full'
+            get 'sitemap.xml', to: 'page#sitemap'
             get 'search', to: 'page#search'
             get '*url', to: 'page#show'
           end
         end
       end
 
-      class PageController < ActionController::Base
-        layout :docs_layout
-
-        def configuration
-          params[:configuration] || Trifle::Docs.default
-        end
-
-        def docs_layout
-          "layouts/trifle/docs/#{configuration.layout}"
-        end
+      module PageControllerHelpers
+        private
 
-        def show # rubocop:disable Metrics/AbcSize
-          url = [params[:url], params[:format]].compact.join('.')
+        def render_llms(url, allow_empty: false)
           meta = Trifle::Docs.meta(url: url, config: configuration)
-          render_not_found and return if meta.nil?
+          return send_file(meta['path']) if meta && meta['type'] == 'file'
 
-          if Trifle::Docs::Helper::AiDetection.ai_scraper?(request.user_agent) && meta['type'] != 'file'
-            render_markdown(url: url, meta: meta)
-            return
-          end
-          render_file(meta: meta) and return if meta['type'] == 'file'
+          content = yield
+          return render_not_found if content.nil?
+          return render_not_found if !allow_empty && content.strip.empty?
 
-          render_content(url: url, meta: meta)
+          render plain: content, content_type: 'text/markdown'
         end
 
-        def search
-          results = Trifle::Docs.search(query: params[:query], scope: params[:scope])
+        def render_sitemap(url)
+          meta = Trifle::Docs.meta(url: url, config: configuration)
+          return send_file(meta['path']) if meta && meta['type'] == 'file'
 
-          render 'search', locals: {
-            results: results,
-            query: params[:query],
-            scope: params[:scope],
-            sitemap: Trifle::Docs.sitemap,
-            meta: { description: 'Search' }
-          }
+          content = yield
+          return render_not_found if content.nil? || content.strip.empty?
+
+          render plain: content, content_type: 'application/xml'
         end
 
         def render_not_found
@@ -78,13 +68,100 @@ if Object.const_defined?('Rails')
           }
         end
 
-        def render_markdown
+        def render_markdown(url:, meta:)
           render plain: Trifle::Docs::Helper::MarkdownLayout.render(
             meta: meta,
             raw_content: Trifle::Docs.raw_content(url: url, config: configuration),
             sitemap: Trifle::Docs.sitemap(config: configuration)
           ), content_type: 'text/markdown'
         end
+
+        def fetch_meta(url)
+          Trifle::Docs.meta(url: url, config: configuration)
+        end
+
+        def render_for_meta(meta, url, wants_md, request)
+          return render_markdown(url: url, meta: meta) if render_markdown?(meta, wants_md, request)
+          return render_file(meta: meta) if file_meta?(meta)
+
+          render_content(url: url, meta: meta)
+        end
+
+        def file_meta?(meta)
+          meta['type'] == 'file'
+        end
+
+        def resolve_url(params)
+          raw_url = params[:url]
+          format = params[:format]
+          wants_md = markdown_requested?(format, request)
+          url = wants_md ? raw_url : [raw_url, format].compact.join('.')
+          [url, wants_md]
+        end
+
+        def render_markdown?(meta, wants_md, request)
+          return false if meta['type'] == 'file'
+
+          wants_md || Trifle::Docs::Helper::AiDetection.ai_scraper?(request.user_agent)
+        end
+
+        def markdown_requested?(format, request)
+          return true if format.to_s.downcase == 'md'
+
+          request.headers['Accept'].to_s.include?('text/markdown')
+        end
+      end
+
+      class PageController < ActionController::Base
+        include PageControllerHelpers
+
+        layout :docs_layout
+
+        def configuration
+          params[:configuration] || Trifle::Docs.default
+        end
+
+        def docs_layout
+          "layouts/trifle/docs/#{configuration.layout}"
+        end
+
+        def show
+          url, wants_md = resolve_url(params)
+          meta = fetch_meta(url)
+          return render_not_found if meta.nil?
+
+          render_for_meta(meta, url, wants_md, request)
+        end
+
+        def search
+          results = Trifle::Docs.search(query: params[:query], scope: params[:scope])
+
+          render 'search', locals: {
+            results: results,
+            query: params[:query],
+            scope: params[:scope],
+            sitemap: Trifle::Docs.sitemap,
+            meta: { description: 'Search' }
+          }
+        end
+
+        def llms
+          render_llms('llms.txt', allow_empty: true) do
+            Trifle::Docs::Helper::Llms.homepage_markdown(config: configuration)
+          end
+        end
+
+        def llms_full
+          render_llms('llms-full.txt') do
+            Trifle::Docs::Helper::Llms.full_markdown(config: configuration)
+          end
+        end
+
+        def sitemap
+          render_sitemap('sitemap.xml') do
+            Trifle::Docs::Helper::Sitemap.xml(config: configuration)
+          end
+        end
       end
     end
   end

data/lib/trifle/docs/helper/llms.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Trifle
+  module Docs
+    module Helper
+      module Llms
+        module_function
+
+        def homepage_markdown(config: nil)
+          meta = Trifle::Docs.meta(url: '', config: config)
+          return nil if meta.nil?
+
+          Trifle::Docs::Helper::MarkdownLayout.render(
+            meta: meta,
+            raw_content: Trifle::Docs.raw_content(url: '', config: config),
+            sitemap: Trifle::Docs.sitemap(config: config)
+          )
+        end
+
+        def full_markdown(config: nil)
+          sitemap = Trifle::Docs.sitemap(config: config)
+          pages = flatten_sitemap(sitemap)
+
+          chunks = pages.filter_map do |page|
+            meta = page[:meta]
+            next if meta.nil? || meta['type'] == 'file'
+
+            raw = Trifle::Docs.raw_content(url: page[:url], config: config)
+            next if raw.nil? || raw.strip.empty?
+
+            format_page(meta: meta, url: page[:url], raw_content: raw)
+          end
+
+          chunks.join("\n\n")
+        end
+
+        def flatten_sitemap(node, path = [])
+          return [] unless node.is_a?(Hash)
+
+          entries = []
+          meta = node['_meta']
+          entries << { url: path.join('/'), meta: meta } if meta
+
+          node.keys.reject { |key| key == '_meta' }.sort.each do |key|
+            entries.concat(flatten_sitemap(node[key], path + [key]))
+          end
+
+          entries
+        end
+
+        def format_page(meta:, url:, raw_content:)
+          lines = page_header(meta: meta, url: url)
+          lines << raw_content.to_s.strip
+          lines << ''
+          lines << '---'
+
+          lines.join("\n")
+        end
+
+        def page_header(meta:, url:)
+          title = meta['title'] || Trifle::Docs::Helper::MarkdownLayout.derive_title_from_url(url)
+          page_url = meta['url'] || "/#{url}"
+
+          [
+            "# #{title}",
+            '',
+            "Source: #{page_url}",
+            ''
+          ]
+        end
+      end
+    end
+  end
+end

data/lib/trifle/docs/helper/sitemap.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'cgi'
+require 'time'
+
+module Trifle
+  module Docs
+    module Helper
+      module Sitemap
+        module_function
+
+        def xml(config: nil)
+          sitemap = Trifle::Docs.sitemap(config: config)
+          urls = sitemap_urls(sitemap)
+          return nil if urls.empty?
+
+          build_document(urls)
+        end
+
+        def flatten_sitemap(node, path = [])
+          return [] unless node.is_a?(Hash)
+
+          entries = []
+          meta = node['_meta']
+          entries << { url: path.join('/'), meta: meta } if meta
+
+          node.keys.reject { |key| key == '_meta' }.sort.each do |key|
+            entries.concat(flatten_sitemap(node[key], path + [key]))
+          end
+
+          entries
+        end
+
+        def sitemap_urls(sitemap)
+          entries = flatten_sitemap(sitemap)
+          entries.filter_map do |entry|
+            meta = entry[:meta]
+            next if meta.nil? || meta['type'] == 'file'
+
+            build_url_entry(entry[:url], meta)
+          end
+        end
+
+        def build_document(urls)
+          [
+            %(<?xml version="1.0" encoding="UTF-8"?>),
+            %(<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">),
+            urls.join("\n"),
+            %(</urlset>)
+          ].join("\n")
+        end
+
+        def build_url_entry(url, meta)
+          loc = normalize_loc(url, meta)
+          lastmod = format_lastmod(meta['updated_at'])
+          lastmod_tag = lastmod ? "<lastmod>#{lastmod}</lastmod>" : ''
+
+          "<url><loc>#{loc}</loc>#{lastmod_tag}</url>"
+        end
+
+        def normalize_loc(url, meta)
+          loc = meta['url']
+          loc = "/#{url}" if loc.nil? || loc.empty?
+          loc = '/' if loc == '//'
+          CGI.escapeHTML(loc)
+        end
+
+        def format_lastmod(value)
+          return nil if value.nil?
+
+          return value.utc.iso8601 if value.respond_to?(:utc) && value.respond_to?(:iso8601)
+          return value.iso8601 if value.respond_to?(:iso8601)
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/trifle/docs/version.rb

@@ -2,6 +2,6 @@
 
 module Trifle
   module Docs
-    VERSION = '0.5.0'
+    VERSION = '0.7.0'
   end
 end

data/lib/trifle/docs.rb

@@ -4,6 +4,8 @@ require_relative 'docs/configuration'
 require_relative 'docs/helper/tree'
 require_relative 'docs/helper/markdown_layout'
 require_relative 'docs/helper/ai_detection'
+require_relative 'docs/helper/llms'
+require_relative 'docs/helper/sitemap'
 require_relative 'docs/harvester'
 require_relative 'docs/harvester/file'
 require_relative 'docs/harvester/markdown'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: trifle-docs
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Jozef Vaclavik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-12-11 00:00:00.000000000 Z
+date: 2026-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -201,7 +201,9 @@ files:
 - lib/trifle/docs/harvester/file.rb
 - lib/trifle/docs/harvester/markdown.rb
 - lib/trifle/docs/helper/ai_detection.rb
+- lib/trifle/docs/helper/llms.rb
 - lib/trifle/docs/helper/markdown_layout.rb
+- lib/trifle/docs/helper/sitemap.rb
 - lib/trifle/docs/helper/tree.rb
 - lib/trifle/docs/operations/collection.rb
 - lib/trifle/docs/operations/content.rb