trifle-docs 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bfae89cd75902acfa8a8b3c23e082e9820e6af13f05a8f69c0e955ae0fe65f56
-  data.tar.gz: db34713f816c8ad860930a85ce198654fcb1adbe91716a308db3a8b11c941fb9
+  metadata.gz: 9f51e6c6b892eaca5b790d652ccb970146c9e98b624cf3353c84af3ffb95ed4c
+  data.tar.gz: f5058c9fa39c81a635db554fabffaab5535f57a76a078138319df0e72732cd49
 SHA512:
-  metadata.gz: ec86cf47abf1b3d8534ae6fce439d353e694fb443c37ac59753e6ca18f7f72330b007e1bdcc1390f0f6b87e3ea89b41ef41db85816059eceb54716bb49ef6d43
-  data.tar.gz: c2775f303aec6dfa94a2071fc1722a932d996d907aeb767f58269ee8060d86d5b49b94c42ce67c28a7060175e365434756b446f3b64896ed02e434d825bfc40e
+  metadata.gz: 48ce9ec16b882be46f90825f0e12bb13c783fa16e5b22f5eae27551418a5e58d112a321496fdf46810474c68de5f8933661931a696b6378bc836ffd7a8e74192
+  data.tar.gz: 9721a213e1f2cef167e0adf4ed18025b0740ba6594e26f6f0f326e4e6412631f5957447f503cd2ca45c4ce4e2df5efb497a15a8aaf2943a29b246a2c373869a8

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    trifle-docs (0.6.0)
+    trifle-docs (0.7.0)
       redcarpet
       rouge
       sinatra

data/lib/trifle/docs/app.rb

@@ -27,9 +27,7 @@ module Trifle
 
       get '/llms.txt' do
         meta = Trifle::Docs.meta(url: 'llms.txt')
-        if meta && meta['type'] == 'file'
-          return send_file(meta['path'])
-        end
+        return send_file(meta['path']) if meta && meta['type'] == 'file'
 
         content = Trifle::Docs::Helper::Llms.homepage_markdown
         halt(404, 'Not Found') if content.nil?
@@ -40,9 +38,7 @@ module Trifle
 
       get '/llms-full.txt' do
         meta = Trifle::Docs.meta(url: 'llms-full.txt')
-        if meta && meta['type'] == 'file'
-          return send_file(meta['path'])
-        end
+        return send_file(meta['path']) if meta && meta['type'] == 'file'
 
         content = Trifle::Docs::Helper::Llms.full_markdown
         halt(404, 'Not Found') if content.nil? || content.strip.empty?
@@ -51,32 +47,70 @@ module Trifle
         content
       end
 
+      get '/sitemap.xml' do
+        meta = Trifle::Docs.meta(url: 'sitemap.xml')
+        return send_file(meta['path']) if meta && meta['type'] == 'file'
+
+        render_generated_sitemap
+      end
+
       get '/*' do
+        handle_request(params, request)
+      end
+
+      def markdown_requested?(request, params)
+        return true if params['format'].to_s.downcase == 'md'
+
+        accept = request.env['HTTP_ACCEPT'].to_s
+        accept.include?('text/markdown')
+      end
+
+      def render_markdown?(meta, request, params)
+        return false if meta['type'] == 'file'
+
+        markdown_requested?(request, params) ||
+          Trifle::Docs::Helper::AiDetection.ai_scraper?(request.user_agent)
+      end
+
+      def handle_request(params, request)
         url = params['splat'].first.chomp('/')
         meta = Trifle::Docs.meta(url: url)
         halt(404, 'Not Found') if meta.nil?
 
-        if Trifle::Docs::Helper::AiDetection.ai_scraper?(request.user_agent) && meta['type'] != 'file'
-          content_type 'text/markdown'
-          return Trifle::Docs::Helper::MarkdownLayout.render(
-            meta: meta,
-            raw_content: Trifle::Docs.raw_content(url: url),
-            sitemap: Trifle::Docs.sitemap
-          )
-        end
-
-        if meta['type'] == 'file'
-          send_file meta['path']
-        else
-          erb (meta['template'] || 'page').to_sym, {}, {
-            sitemap: Trifle::Docs.sitemap,
-            collection: Trifle::Docs.collection(url: url),
-            content: Trifle::Docs.content(url: url),
-            meta: meta,
-            url: url
-          }
-        end
+        return render_markdown(meta, url) if render_markdown?(meta, request, params)
+        return send_file(meta['path']) if meta['type'] == 'file'
+
+        render_html(meta, url)
+      end
+
+      def render_markdown(meta, url)
+        content_type 'text/markdown'
+        Trifle::Docs::Helper::MarkdownLayout.render(
+          meta: meta,
+          raw_content: Trifle::Docs.raw_content(url: url),
+          sitemap: Trifle::Docs.sitemap
+        )
+      end
+
+      def render_html(meta, url)
+        erb (meta['template'] || 'page').to_sym, {}, {
+          sitemap: Trifle::Docs.sitemap,
+          collection: Trifle::Docs.collection(url: url),
+          content: Trifle::Docs.content(url: url),
+          meta: meta,
+          url: url
+        }
       end
+
+      def render_generated_sitemap
+        content = Trifle::Docs::Helper::Sitemap.xml
+        halt(404, 'Not Found') if content.nil? || content.strip.empty?
+
+        content_type 'application/xml'
+        content
+      end
+
+      private :handle_request
     end
   end
 end

data/lib/trifle/docs/engine.rb

@@ -19,79 +19,35 @@ if Object.const_defined?('Rails')
             root to: 'page#show'
             get 'llms.txt', to: 'page#llms'
             get 'llms-full.txt', to: 'page#llms_full'
+            get 'sitemap.xml', to: 'page#sitemap'
             get 'search', to: 'page#search'
             get '*url', to: 'page#show'
           end
         end
       end
 
-      class PageController < ActionController::Base
-        layout :docs_layout
-
-        def configuration
-          params[:configuration] || Trifle::Docs.default
-        end
+      module PageControllerHelpers
+        private
 
-        def docs_layout
-          "layouts/trifle/docs/#{configuration.layout}"
-        end
-
-        def show # rubocop:disable Metrics/AbcSize
-          url = [params[:url], params[:format]].compact.join('.')
+        def render_llms(url, allow_empty: false)
           meta = Trifle::Docs.meta(url: url, config: configuration)
-          render_not_found and return if meta.nil?
-
-          if Trifle::Docs::Helper::AiDetection.ai_scraper?(request.user_agent) && meta['type'] != 'file'
-            render_markdown(url: url, meta: meta)
-            return
-          end
-          render_file(meta: meta) and return if meta['type'] == 'file'
-
-          render_content(url: url, meta: meta)
-        end
-
-        def search
-          results = Trifle::Docs.search(query: params[:query], scope: params[:scope])
-
-          render 'search', locals: {
-            results: results,
-            query: params[:query],
-            scope: params[:scope],
-            sitemap: Trifle::Docs.sitemap,
-            meta: { description: 'Search' }
-          }
-        end
-
-        def llms
-          meta = Trifle::Docs.meta(url: 'llms.txt', config: configuration)
-          if meta && meta['type'] == 'file'
-            send_file(meta['path'])
-            return
-          end
+          return send_file(meta['path']) if meta && meta['type'] == 'file'
 
-          content = Trifle::Docs::Helper::Llms.homepage_markdown(config: configuration)
-          if content.nil?
-            render_not_found
-            return
-          end
+          content = yield
+          return render_not_found if content.nil?
+          return render_not_found if !allow_empty && content.strip.empty?
 
           render plain: content, content_type: 'text/markdown'
         end
 
-        def llms_full
-          meta = Trifle::Docs.meta(url: 'llms-full.txt', config: configuration)
-          if meta && meta['type'] == 'file'
-            send_file(meta['path'])
-            return
-          end
+        def render_sitemap(url)
+          meta = Trifle::Docs.meta(url: url, config: configuration)
+          return send_file(meta['path']) if meta && meta['type'] == 'file'
 
-          content = Trifle::Docs::Helper::Llms.full_markdown(config: configuration)
-          if content.nil? || content.strip.empty?
-            render_not_found
-            return
-          end
+          content = yield
+          return render_not_found if content.nil? || content.strip.empty?
 
-          render plain: content, content_type: 'text/markdown'
+          render plain: content, content_type: 'application/xml'
         end
 
         def render_not_found
@@ -119,6 +75,93 @@ if Object.const_defined?('Rails')
             sitemap: Trifle::Docs.sitemap(config: configuration)
           ), content_type: 'text/markdown'
         end
+
+        def fetch_meta(url)
+          Trifle::Docs.meta(url: url, config: configuration)
+        end
+
+        def render_for_meta(meta, url, wants_md, request)
+          return render_markdown(url: url, meta: meta) if render_markdown?(meta, wants_md, request)
+          return render_file(meta: meta) if file_meta?(meta)
+
+          render_content(url: url, meta: meta)
+        end
+
+        def file_meta?(meta)
+          meta['type'] == 'file'
+        end
+
+        def resolve_url(params)
+          raw_url = params[:url]
+          format = params[:format]
+          wants_md = markdown_requested?(format, request)
+          url = wants_md ? raw_url : [raw_url, format].compact.join('.')
+          [url, wants_md]
+        end
+
+        def render_markdown?(meta, wants_md, request)
+          return false if meta['type'] == 'file'
+
+          wants_md || Trifle::Docs::Helper::AiDetection.ai_scraper?(request.user_agent)
+        end
+
+        def markdown_requested?(format, request)
+          return true if format.to_s.downcase == 'md'
+
+          request.headers['Accept'].to_s.include?('text/markdown')
+        end
+      end
+
+      class PageController < ActionController::Base
+        include PageControllerHelpers
+
+        layout :docs_layout
+
+        def configuration
+          params[:configuration] || Trifle::Docs.default
+        end
+
+        def docs_layout
+          "layouts/trifle/docs/#{configuration.layout}"
+        end
+
+        def show
+          url, wants_md = resolve_url(params)
+          meta = fetch_meta(url)
+          return render_not_found if meta.nil?
+
+          render_for_meta(meta, url, wants_md, request)
+        end
+
+        def search
+          results = Trifle::Docs.search(query: params[:query], scope: params[:scope])
+
+          render 'search', locals: {
+            results: results,
+            query: params[:query],
+            scope: params[:scope],
+            sitemap: Trifle::Docs.sitemap,
+            meta: { description: 'Search' }
+          }
+        end
+
+        def llms
+          render_llms('llms.txt', allow_empty: true) do
+            Trifle::Docs::Helper::Llms.homepage_markdown(config: configuration)
+          end
+        end
+
+        def llms_full
+          render_llms('llms-full.txt') do
+            Trifle::Docs::Helper::Llms.full_markdown(config: configuration)
+          end
+        end
+
+        def sitemap
+          render_sitemap('sitemap.xml') do
+            Trifle::Docs::Helper::Sitemap.xml(config: configuration)
+          end
+        end
       end
     end
   end

data/lib/trifle/docs/helper/llms.rb

@@ -49,20 +49,25 @@ module Trifle
         end
 
         def format_page(meta:, url:, raw_content:)
-          title = meta['title'] || Trifle::Docs::Helper::MarkdownLayout.derive_title_from_url(url)
-          page_url = meta['url'] || "/#{url}"
-
-          lines = []
-          lines << "# #{title}"
-          lines << ''
-          lines << "Source: #{page_url}"
-          lines << ''
+          lines = page_header(meta: meta, url: url)
           lines << raw_content.to_s.strip
           lines << ''
           lines << '---'
 
           lines.join("\n")
         end
+
+        def page_header(meta:, url:)
+          title = meta['title'] || Trifle::Docs::Helper::MarkdownLayout.derive_title_from_url(url)
+          page_url = meta['url'] || "/#{url}"
+
+          [
+            "# #{title}",
+            '',
+            "Source: #{page_url}",
+            ''
+          ]
+        end
       end
     end
   end

data/lib/trifle/docs/helper/sitemap.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'cgi'
+require 'time'
+
+module Trifle
+  module Docs
+    module Helper
+      module Sitemap
+        module_function
+
+        def xml(config: nil)
+          sitemap = Trifle::Docs.sitemap(config: config)
+          urls = sitemap_urls(sitemap)
+          return nil if urls.empty?
+
+          build_document(urls)
+        end
+
+        def flatten_sitemap(node, path = [])
+          return [] unless node.is_a?(Hash)
+
+          entries = []
+          meta = node['_meta']
+          entries << { url: path.join('/'), meta: meta } if meta
+
+          node.keys.reject { |key| key == '_meta' }.sort.each do |key|
+            entries.concat(flatten_sitemap(node[key], path + [key]))
+          end
+
+          entries
+        end
+
+        def sitemap_urls(sitemap)
+          entries = flatten_sitemap(sitemap)
+          entries.filter_map do |entry|
+            meta = entry[:meta]
+            next if meta.nil? || meta['type'] == 'file'
+
+            build_url_entry(entry[:url], meta)
+          end
+        end
+
+        def build_document(urls)
+          [
+            %(<?xml version="1.0" encoding="UTF-8"?>),
+            %(<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">),
+            urls.join("\n"),
+            %(</urlset>)
+          ].join("\n")
+        end
+
+        def build_url_entry(url, meta)
+          loc = normalize_loc(url, meta)
+          lastmod = format_lastmod(meta['updated_at'])
+          lastmod_tag = lastmod ? "<lastmod>#{lastmod}</lastmod>" : ''
+
+          "<url><loc>#{loc}</loc>#{lastmod_tag}</url>"
+        end
+
+        def normalize_loc(url, meta)
+          loc = meta['url']
+          loc = "/#{url}" if loc.nil? || loc.empty?
+          loc = '/' if loc == '//'
+          CGI.escapeHTML(loc)
+        end
+
+        def format_lastmod(value)
+          return nil if value.nil?
+
+          return value.utc.iso8601 if value.respond_to?(:utc) && value.respond_to?(:iso8601)
+          return value.iso8601 if value.respond_to?(:iso8601)
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/trifle/docs/version.rb

@@ -2,6 +2,6 @@
 
 module Trifle
   module Docs
-    VERSION = '0.6.0'
+    VERSION = '0.7.0'
   end
 end

data/lib/trifle/docs.rb

@@ -5,6 +5,7 @@ require_relative 'docs/helper/tree'
 require_relative 'docs/helper/markdown_layout'
 require_relative 'docs/helper/ai_detection'
 require_relative 'docs/helper/llms'
+require_relative 'docs/helper/sitemap'
 require_relative 'docs/harvester'
 require_relative 'docs/harvester/file'
 require_relative 'docs/harvester/markdown'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: trifle-docs
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Jozef Vaclavik
@@ -203,6 +203,7 @@ files:
 - lib/trifle/docs/helper/ai_detection.rb
 - lib/trifle/docs/helper/llms.rb
 - lib/trifle/docs/helper/markdown_layout.rb
+- lib/trifle/docs/helper/sitemap.rb
 - lib/trifle/docs/helper/tree.rb
 - lib/trifle/docs/operations/collection.rb
 - lib/trifle/docs/operations/content.rb