trifle-docs 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 539c91d9c90586bfa38c2b7d89bbdf4dbbfde706c8f1ac306eb57aea550d3794
-  data.tar.gz: 784e68bf02e62481f5269bd6e6ed10f2c2bfc41857346466d6a117ee5f9a0f09
+  metadata.gz: b52d0fa673ad7999994caf2fda33169ac1815587cbf3023cce7a3a5ec65373a2
+  data.tar.gz: 56c8c894a50aef901915dee9eab80a2f7b7e5a6fe3b46cbe344f4360f0c639b1
 SHA512:
-  metadata.gz: e2f827a9b25d6ef96a89c6e1cb73c7e20da8e3ab7d318be7ea3c89f78a4116ac90650b3628c268957fd4042f824418d3f53f150b1c886c851afab35b538b9461
-  data.tar.gz: ad2b341f5e35061ee9c7e78775135610a3bf4067594fcfa26ca9df604c77b27d093d0eef78d02208679b41810a26083562a43e88d73cd1145364c4b3a3cc31c4
+  metadata.gz: 59e6d106be4c79000a4470429308ce833ba8a6e0e3d02d8b1bdfacafd638030e14687b53aa5f5d81f55b91158b8873d9eb624bc0665d04d3bb781afc31927c8d
+  data.tar.gz: d7ca56cd6809bd75891a5bf6b8fd49847007539dced62b5c69b43d9c779e00d963c3986fba84ff457d68edd52947cc5a554333584325ac56bb42e2f95d4e5579

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    trifle-docs (0.3.0)
+    trifle-docs (0.5.0)
       redcarpet
       rouge
       sinatra
@@ -11,25 +11,29 @@ GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
+    base64 (0.3.0)
     byebug (11.1.3)
     diff-lcs (1.5.0)
-    mustermann (2.0.2)
+    mustermann (3.0.3)
       ruby2_keywords (~> 0.0.1)
-    nio4r (2.5.8)
+    nio4r (2.7.4)
     parallel (1.22.1)
     parser (3.1.2.0)
       ast (~> 2.4.1)
-    puma (5.6.4)
+    puma (6.6.0)
       nio4r (~> 2.0)
-    rack (2.2.4)
-    rack-protection (2.2.2)
-      rack
+    rack (3.1.16)
+    rack-protection (4.0.0)
+      base64 (>= 0.1.0)
+      rack (>= 3.0.0, < 4)
+    rack-session (2.0.0)
+      rack (>= 3.0.0)
     rainbow (3.1.1)
     rake (13.0.6)
-    redcarpet (3.5.1)
+    redcarpet (3.6.0)
     regexp_parser (2.5.0)
     rexml (3.2.5)
-    rouge (4.0.0)
+    rouge (4.2.0)
     rspec (3.11.0)
       rspec-core (~> 3.11.0)
       rspec-expectations (~> 3.11.0)
@@ -56,17 +60,19 @@ GEM
       parser (>= 3.1.1.0)
     ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.5)
-    sinatra (2.2.2)
-      mustermann (~> 2.0)
-      rack (~> 2.2)
-      rack-protection (= 2.2.2)
+    sinatra (4.0.0)
+      mustermann (~> 3.0)
+      rack (>= 3.0.0, < 4)
+      rack-protection (= 4.0.0)
+      rack-session (>= 2.0.0, < 3)
       tilt (~> 2.0)
-    tilt (2.0.11)
+    tilt (2.3.0)
     unicode-display_width (2.1.0)
-    yaml (0.2.0)
+    yaml (0.3.0)
 
 PLATFORMS
   arm64-darwin-21
+  arm64-darwin-24
   x86_64-linux
 
 DEPENDENCIES

data/README.md

@@ -1,53 +1,79 @@
 # Trifle::Docs
 
-[![Gem Version](https://badge.fury.io/rb/trifle-docs.svg)](https://badge.fury.io/rb/trifle-docs)
-![Ruby](https://github.com/trifle-io/trifle-docs/workflows/Ruby/badge.svg?branch=main)
-[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/trifle-io/trifle-docs)
+[![Gem Version](https://badge.fury.io/rb/trifle-docs.svg)](https://rubygems.org/gems/trifle-docs)
+[![Ruby](https://github.com/trifle-io/trifle-docs/workflows/Ruby/badge.svg?branch=main)](https://github.com/trifle-io/trifle-docs)
 
-Simple documentation backend for your markdown files.
+Simple router for your static documentation. Like markdown, or textile, or whatever files. It maps your docs folder structure into URLs and renders them within the simplest template possible.
 
-Integrate your documentation or blog into your existing Rails application. `Trifle::Docs` maps your docs folder full of markdown files to your URLs and allows you to integrate it with same layout as the rest of your app.
+## Documentation
 
-![Demo App](demo.gif)
+For comprehensive guides, API reference, and examples, visit [trifle.io/trifle-docs](https://trifle.io/trifle-docs)
 
+![Demo App](demo.gif)
 
-## Documentation
+## Installation
 
-You can find guides and documentation at https://trifle.io/trifle-docs
+Add this line to your application's Gemfile:
 
-## Installation
+```ruby
+gem 'trifle-docs'
+```
 
-Install the gem and add to the application's Gemfile by executing:
+And then execute:
 
-```sh
-$ bundle add trifle-docs
+```bash
+$ bundle install
 ```
 
-If bundler is not being used to manage dependencies, install the gem by executing:
+Or install it yourself as:
 
-```sh
+```bash
 $ gem install trifle-docs
 ```
 
-## Usage
+## Quick Start
 
-You can use this as a build-in Sinatra app or mount it in your Rails app. For each usecse, refere to documentation. Below is sample Sinatra integration.
+### 1. Configure
 
 ```ruby
-# app.rb
 require 'trifle/docs'
 
 Trifle::Docs.configure do |config|
-  config.path = 'docs'
-  config.templates = File.join(__dir__, '..', 'templates', 'simple')
+  config.path = File.join(__dir__, 'docs')
+  config.views = File.join(__dir__, 'templates')
   config.register_harvester(Trifle::Docs::Harvester::Markdown)
   config.register_harvester(Trifle::Docs::Harvester::File)
 end
+```
+
+### 2. Create documentation structure
 
-Trifle::Docs.App.run!
 ```
+docs/
+├── index.md
+├── getting-started/
+│   ├── index.md
+│   └── installation.md
+└── api/
+    ├── index.md
+    └── reference.md
+```
+
+### 3. Use in your application
 
-### Templates
+```ruby
+# As Rack middleware
+use Trifle::Docs::Middleware
+
+# Or mount in Rails
+Rails.application.routes.draw do
+  mount Trifle::Docs::Engine => '/docs'
+end
+
+# Or Sinatra app
+```
+
+### 4. Templates
 
 Please create two files in folder you provided the configuration.
 
@@ -67,7 +93,7 @@ Please create two files in folder you provided the configuration.
 <%= content %>
 ```
 
-### Template variables
+#### Template variables
 
 There are several variables available in your template file (except `layout.erb`).
 - `sitemap` - complete sitemap tree of the folder.
@@ -75,22 +101,43 @@ There are several variables available in your template file (except `layout.erb`
 - `content` - rendered markdown file.
 - `meta` - metadata from markdown file.
 
-## Development
+## Features
+
+- **File-based routing** - Maps folder structure to URL paths
+- **Multiple harvesters** - Markdown, textile, and custom file processors
+- **Template system** - ERB templates with layout support
+- **Flexible integration** - Works with Rack, Rails, Sinatra
+- **Caching support** - Optional caching for production environments
+- **Navigation helpers** - Automatic menu and breadcrumb generation
+
+## Harvesters
+
+Trifle::Docs supports multiple content processors:
+
+- **Markdown** - Process `.md` files with frontmatter support
+- **File** - Handle static assets and non-markdown content
+- **Custom** - Build your own harvesters for specialized content
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+## Testing
 
-You can test the sinatra app by running `bin/docs` that uses `templates/simple` templates to render `docs` files.
+Tests focus on documenting behavior and ensuring reliability. To run the test suite:
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+```bash
+$ bundle exec rspec
+```
+
+Tests are meant to be **simple and isolated**. Every test should be **independent** and able to run in any order. Tests should be **self-contained** and set up their own configuration.
+
+Use **single layer testing** to focus on testing a specific class or module in isolation. Use **appropriate stubbing** for file system operations when testing harvesters and routing logic.
+
+**Repeat yourself** in test setup for clarity rather than complex shared setups that can hide dependencies.
+
+Tests verify that file system changes are properly reflected in the documentation routing and that templates render correctly with provided content.
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/trifle-io/trifle-docs. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/trifle-io/trifle-docs/blob/master/CODE_OF_CONDUCT.md).
+Bug reports and pull requests are welcome on GitHub at https://github.com/trifle-io/trifle-docs.
 
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
-
-## Code of Conduct
-
-Everyone interacting in the Trifle::Docs project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/trifle-io/trifle-docs/blob/master/CODE_OF_CONDUCT.md).

data/lib/trifle/docs/app.rb

@@ -10,11 +10,35 @@ module Trifle
         set :views, proc { Trifle::Docs.default.views }
       end
 
+      get '/search' do
+        results = Trifle::Docs.search(query: params['query'], scope: params['scope'])
+        erb(
+          'search'.to_sym,
+          {},
+          {
+            results: results,
+            query: params['query'],
+            scope: params['scope'],
+            sitemap: Trifle::Docs.sitemap,
+            meta: { description: 'Search' }
+          }
+        )
+      end
+
       get '/*' do
         url = params['splat'].first.chomp('/')
         meta = Trifle::Docs.meta(url: url)
         halt(404, 'Not Found') if meta.nil?
 
+        if Trifle::Docs::Helper::AiDetection.ai_scraper?(request.user_agent) && meta['type'] != 'file'
+          content_type 'text/markdown'
+          return Trifle::Docs::Helper::MarkdownLayout.render(
+            meta: meta,
+            raw_content: Trifle::Docs.raw_content(url: url),
+            sitemap: Trifle::Docs.sitemap
+          )
+        end
+
         if meta['type'] == 'file'
           send_file meta['path']
         else

data/lib/trifle/docs/engine.rb

@@ -17,6 +17,7 @@ if Object.const_defined?('Rails')
         def self.draw
           Trifle::Docs::Engine.routes.draw do
             root to: 'page#show'
+            get 'search', to: 'page#search'
             get '*url', to: 'page#show'
           end
         end
@@ -33,15 +34,32 @@ if Object.const_defined?('Rails')
           "layouts/trifle/docs/#{configuration.layout}"
         end
 
-        def show
+        def show # rubocop:disable Metrics/AbcSize
           url = [params[:url], params[:format]].compact.join('.')
           meta = Trifle::Docs.meta(url: url, config: configuration)
           render_not_found and return if meta.nil?
+
+          if Trifle::Docs::Helper::AiDetection.ai_scraper?(request.user_agent) && meta['type'] != 'file'
+            render_markdown(url: url, meta: meta)
+            return
+          end
           render_file(meta: meta) and return if meta['type'] == 'file'
 
           render_content(url: url, meta: meta)
         end
 
+        def search
+          results = Trifle::Docs.search(query: params[:query], scope: params[:scope])
+
+          render 'search', locals: {
+            results: results,
+            query: params[:query],
+            scope: params[:scope],
+            sitemap: Trifle::Docs.sitemap,
+            meta: { description: 'Search' }
+          }
+        end
+
         def render_not_found
           render text: 'Not Found', status: 404
         end
@@ -59,6 +77,14 @@ if Object.const_defined?('Rails')
             url: url
           }
         end
+
+        def render_markdown
+          render plain: Trifle::Docs::Helper::MarkdownLayout.render(
+            meta: meta,
+            raw_content: Trifle::Docs.raw_content(url: url, config: configuration),
+            sitemap: Trifle::Docs.sitemap(config: configuration)
+          ), content_type: 'text/markdown'
+        end
       end
     end
   end

data/lib/trifle/docs/harvester/markdown.rb

@@ -33,19 +33,38 @@ module Trifle
               Render.new(with_toc_data: true),
               fenced_code_blocks: true,
               disable_indented_code_blocks: true,
-              footnotes: true
+              footnotes: true,
+              tables: true
             ).render(data.sub(/^---(.*?)---(\s*)/m, ''))
+          rescue StandardError => e
+            puts "Markdown: Failed to parse CONTENT for #{file}: #{e}"
           end
 
-          def meta
-            @meta = nil unless cache
+          def raw_content
+            @raw_content = nil unless cache
+
+            @raw_content ||= data.sub(/^---(.*?)---(\s*)/m, '').strip
+          rescue StandardError => e
+            puts "Markdown: Failed to load RAW CONTENT for #{file}: #{e}"
+          end
 
-            @meta ||= (YAML.safe_load(data[/^---(.*?)---(\s*)/m].to_s) || {}).merge(
+          def default_meta
+            {
               'url' => "/#{[namespace, url].compact.join('/')}",
               'breadcrumbs' => url.split('/'),
               'toc' => toc,
               'updated_at' => ::File.stat(file).mtime
-            )
+            }
+          end
+
+          def meta
+            @meta = nil unless cache
+
+            @meta ||= (YAML.safe_load(data[/^---(.*?)---(\s*)/m].to_s) || {}).merge(default_meta)
+          rescue StandardError => e
+            puts "Markdown: Failed to parse META for #{file}: #{e}"
+
+            default_meta
           end
 
           def toc
@@ -54,6 +73,18 @@ module Trifle
             @toc ||= Redcarpet::Markdown.new(
               Redcarpet::Render::HTML_TOC
             ).render(data.sub(/^---(.*?)---(\s*)/m, ''))
+          rescue StandardError => e
+            puts "Markdown: Failed to parse TOC for #{file}: #{e}"
+          end
+
+          private
+
+          def preload_cache
+            data
+            content
+            raw_content
+            meta
+            toc
           end
         end
       end

data/lib/trifle/docs/harvester.rb

@@ -3,7 +3,7 @@
 module Trifle
   module Docs
     module Harvester
-      class Walker
+      class Walker # rubocop:disable Metrics/ClassLength
         attr_reader :path, :router, :namespace, :cache
 
         def initialize(**keywords)
@@ -41,6 +41,32 @@ module Trifle
           end
         end
 
+        def search_for(query:, scope: nil, limit: 10) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+          return [] if query.nil? || query.strip.empty?
+
+          query_terms = [query.downcase.strip]
+          matches = []
+
+          searchable_routes = filter_searchable_routes(scope)
+
+          searchable_routes.each do |url, conveyor|
+            score = calculate_fuzzy_match_score(conveyor, query_terms)
+            next if score.zero?
+
+            matches << {
+              url: url,
+              conveyor: conveyor,
+              score: score
+            }
+
+            break if matches.size >= limit * 2
+          end
+
+          matches.sort_by { |match| -match[:score] }
+                 .first(limit)
+                 .map { |match| build_search_result(match, query_terms) }
+        end
+
         def collection_for(url:)
           return sitemap if url.empty?
 
@@ -51,6 +77,17 @@ module Trifle
           route_for(url: url)&.content
         end
 
+        def raw_content_for(url:)
+          conveyor = route_for(url: url)
+          return unless conveyor
+
+          if conveyor.respond_to?(:raw_content)
+            conveyor.raw_content
+          elsif conveyor.respond_to?(:content)
+            conveyor.content
+          end
+        end
+
         def meta_for(url:)
           route_for(url: url)&.meta
         end
@@ -62,6 +99,245 @@ module Trifle
         def not_found(url:)
           puts "No route found for url: #{url}"
         end
+
+        private
+
+        def filter_searchable_routes(scope)
+          router.select do |url, conveyor|
+            # Only include searchable harvesters (exclude File harvester)
+            searchable_conveyor?(conveyor) && (scope.nil? || url.start_with?(scope))
+          end
+        end
+
+        def searchable_conveyor?(conveyor)
+          # Include only conveyors that have searchable content (exclude File harvester)
+          conveyor.respond_to?(:content) && conveyor.respond_to?(:meta) &&
+            !conveyor.class.name.include?('File')
+        end
+
+        def calculate_fuzzy_match_score(conveyor, query_terms) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+          score = 0
+          searchable_content = extract_searchable_content(conveyor)
+          query = query_terms.first
+
+          # Fuzzy matching with different strategies
+          searchable_content.each do |field, content|
+            next if content.nil? || content.empty?
+
+            field_weight = get_field_weight(field)
+
+            # Exact match (highest score)
+            exact_matches = content.scan(/#{Regexp.escape(query)}/i).size
+            score += exact_matches * field_weight * 10
+
+            # Subsequence match (fzf-like)
+            score += field_weight * 5 if subsequence_match?(content, query)
+
+            # N-gram similarity
+            ngram_score = calculate_ngram_similarity(content, query)
+            score += (ngram_score * field_weight * 3).to_i
+
+            # Word boundary matches
+            word_matches = content.downcase.scan(/\b#{Regexp.escape(query)}\b/i).size
+            score += word_matches * field_weight * 8
+          end
+
+          score
+        end
+
+        def calculate_match_score(conveyor, query_terms) # rubocop:disable Metrics/AbcSize
+          score = 0
+
+          searchable_content = extract_searchable_content(conveyor)
+
+          query_terms.each do |term|
+            score += search_in_field(searchable_content[:title], term, weight: 10)
+            score += search_in_field(searchable_content[:url], term, weight: 8)
+            score += search_in_field(searchable_content[:tags], term, weight: 7)
+            score += search_in_field(searchable_content[:content], term, weight: 1)
+            score += search_in_field(searchable_content[:meta], term, weight: 5)
+          end
+
+          score
+        end
+
+        def extract_searchable_content(conveyor) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+          content = {}
+
+          content[:url] = conveyor.url.downcase
+
+          if conveyor.respond_to?(:content) && conveyor.cache
+            content[:content] = strip_html(conveyor.content || '').downcase
+          end
+
+          if conveyor.respond_to?(:meta) && conveyor.cache
+            meta = conveyor.meta || {}
+            content[:title] = (meta['title'] || '').downcase
+            content[:meta] = meta.values.join(' ').downcase
+
+            # Include tags as searchable content
+            tags = extract_tags(conveyor)
+            content[:tags] = tags.join(' ').downcase
+          end
+
+          content
+        end
+
+        def search_in_field(field_content, term, weight:)
+          return 0 if field_content.nil? || field_content.empty?
+
+          occurrences = field_content.scan(/#{Regexp.escape(term)}/i).size
+          occurrences * weight
+        end
+
+        def build_search_result(match, query_terms) # rubocop:disable Metrics/MethodLength
+          conveyor = match[:conveyor]
+          title = extract_title(conveyor)
+          excerpt = generate_excerpt(conveyor, query_terms)
+          tags = extract_tags(conveyor)
+
+          {
+            url: match[:url],
+            title: title,
+            excerpt: excerpt,
+            tags: tags,
+            score: match[:score]
+          }
+        end
+
+        def extract_title(conveyor) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+          if conveyor.respond_to?(:meta) && conveyor.cache
+            meta = conveyor.meta || {}
+            title = meta['title']
+            return title unless title.nil? || title.strip.empty?
+          end
+
+          conveyor.url.split('/').last&.gsub(/[-_]/, ' ')&.split&.map(&:capitalize)&.join(' ') || 'Untitled'
+        end
+
+        def extract_tags(conveyor) # rubocop:disable Metrics/MethodLength
+          return [] unless conveyor.respond_to?(:meta) && conveyor.cache
+
+          meta = conveyor.meta || {}
+          tags = meta['tags']
+
+          case tags
+          when Array
+            tags.compact.map(&:to_s)
+          when String
+            [tags]
+          else
+            []
+          end
+        end
+
+        def generate_excerpt(conveyor, query_terms, max_length: 200)
+          return nil unless conveyor.respond_to?(:content) && conveyor.cache
+
+          content = strip_html(conveyor.content || '')
+          return nil if content.empty?
+
+          best_excerpt = find_best_excerpt(content, query_terms, max_length)
+          best_excerpt || content[0, max_length] + (content.length > max_length ? '...' : '')
+        end
+
+        def find_best_excerpt(content, query_terms, max_length) # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity, Metrics/MethodLength
+          return nil if query_terms.empty?
+
+          first_match_pos = nil
+          matched_term = nil
+
+          query_terms.each do |term|
+            pos = content.downcase.index(term.downcase)
+            if pos && (first_match_pos.nil? || pos < first_match_pos)
+              first_match_pos = pos
+              matched_term = term
+            end
+          end
+
+          return nil unless first_match_pos
+
+          context_size = (max_length - matched_term.length) / 2
+          start_pos = [first_match_pos - context_size, 0].max
+          end_pos = [start_pos + max_length, content.length].min
+
+          start_pos = find_word_boundary(content, start_pos, :backward)
+          end_pos = find_word_boundary(content, end_pos, :forward)
+
+          excerpt = content[start_pos...end_pos]
+
+          excerpt = "...#{excerpt}" if start_pos.positive?
+          excerpt += '...' if end_pos < content.length
+
+          excerpt
+        end
+
+        def find_word_boundary(content, pos, direction) # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+          return pos if pos <= 0 || pos >= content.length
+
+          if direction == :backward
+            pos -= 1 while pos.positive? && !content[pos].match?(/\s/)
+            pos += 1 if content[pos].match?(/\s/)
+          else # :forward
+            pos += 1 while pos < content.length && !content[pos].match?(/\s/)
+          end
+
+          pos
+        end
+
+        def get_field_weight(field)
+          case field
+          when :title then 10
+          when :url then 8
+          when :tags then 7
+          when :meta then 5
+          when :content then 1
+          else 1
+          end
+        end
+
+        def subsequence_match?(text, pattern)
+          # fzf-like subsequence matching
+          text_idx = 0
+          pattern_idx = 0
+
+          while text_idx < text.length && pattern_idx < pattern.length
+            pattern_idx += 1 if text[text_idx].downcase == pattern[pattern_idx].downcase
+            text_idx += 1
+          end
+
+          pattern_idx == pattern.length
+        end
+
+        def calculate_ngram_similarity(text, pattern, n: 2) # rubocop:disable Naming/MethodParameterName
+          return 0 if text.length < n || pattern.length < n
+
+          text_ngrams = get_ngrams(text.downcase, n)
+          pattern_ngrams = get_ngrams(pattern.downcase, n)
+
+          return 0 if text_ngrams.empty? || pattern_ngrams.empty?
+
+          intersection = text_ngrams & pattern_ngrams
+          union = text_ngrams | pattern_ngrams
+
+          intersection.length.to_f / union.length
+        end
+
+        def get_ngrams(text, n) # rubocop:disable Naming/MethodParameterName
+          return [] if text.length < n
+
+          (0..text.length - n).map do |i|
+            text[i, n]
+          end
+        end
+
+        def strip_html(html)
+          return '' if html.nil?
+
+          # Ensure valid UTF-8 encoding
+          html = html.encode('UTF-8', invalid: :replace, undef: :replace, replace: '')
+          html.gsub(/<[^>]*>/, ' ').gsub(/\s+/, ' ').strip
+        end
       end
 
       class Sieve
@@ -89,12 +365,18 @@ module Trifle
           @url = url
           @namespace = namespace
           @cache = cache
+
+          preload_cache if cache
+        end
+
+        def preload_cache
+          # NOTE: harvester is responsible for cache implementation
         end
 
         def data
           @data = nil unless cache
 
-          @data ||= ::File.read(file)
+          @data ||= ::File.read(file, encoding: 'utf-8')
         end
       end
     end

data/lib/trifle/docs/helper/ai_detection.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Trifle
+  module Docs
+    module Helper
+      module AiDetection
+        AI_SCRAPER_PATTERNS = [
+          /GPTBot/i,
+          /ChatGPT/i,
+          /ClaudeBot/i,
+          /Claude-Web/i,
+          /anthropic/i,
+          /Perplexity/i,
+          /Google-Extended/i,
+          /CCBot/i,
+          /AI2Bot/i,
+          /FacebookBot/i
+        ].freeze
+
+        module_function
+
+        def ai_scraper?(user_agent)
+          return false if user_agent.nil? || user_agent.empty?
+
+          AI_SCRAPER_PATTERNS.any? { |pattern| user_agent.match?(pattern) }
+        end
+      end
+    end
+  end
+end

data/lib/trifle/docs/helper/markdown_layout.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Trifle
+  module Docs
+    module Helper
+      module MarkdownLayout
+        module_function
+
+        def render(meta:, raw_content:, sitemap:) # rubocop:disable Metrics/MethodLength
+          lines = []
+          title = meta['title'] || derive_title_from_url(meta['url'])
+
+          lines << "# #{title}"
+          lines << ''
+          lines << '## Navigation'
+          lines << navigation_toc(sitemap)
+          lines << ''
+          lines << '## Content'
+          lines << raw_content.to_s.strip
+          lines << ''
+
+          lines.join("\n")
+        end
+
+        def navigation_toc(sitemap, depth: 0) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+          return '' unless sitemap.is_a?(Hash)
+
+          sitemap.keys.reject { |k| k == '_meta' }.sort.map do |key|
+            node = sitemap[key]
+            meta = node['_meta'] || {}
+            title = meta['title'] || derive_title_from_url(meta['url'] || key)
+            url = meta['url'] || "/#{key}"
+            indent = '  ' * depth
+            children = node.reject { |child_key, _| child_key == '_meta' }
+
+            [
+              "#{indent}- [#{title}](#{url})",
+              navigation_toc(children, depth: depth + 1)
+            ].reject(&:empty?).join("\n")
+          end.join("\n")
+        end
+
+        def derive_title_from_url(url)
+          return 'Untitled' if url.nil? || url.empty?
+
+          url.split('/').last.to_s.gsub(/[-_]/, ' ').split.map(&:capitalize).join(' ')
+        end
+      end
+    end
+  end
+end

data/lib/trifle/docs/operations/raw_content.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Trifle
+  module Docs
+    module Operations
+      class RawContent
+        attr_reader :url
+
+        def initialize(**keywords)
+          @url = keywords.fetch(:url)
+          @config = keywords[:config]
+        end
+
+        def config
+          @config || Trifle::Docs.default
+        end
+
+        def perform
+          config.harvester.raw_content_for(url: url)
+        end
+      end
+    end
+  end
+end

data/lib/trifle/docs/operations/search.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Trifle
+  module Docs
+    module Operations
+      class Search
+        attr_reader :query, :scope
+
+        def initialize(**keywords)
+          @query = keywords.fetch(:query)
+          @scope = keywords[:scope] # Optional scope to limit search to subfolder
+          @config = keywords[:config]
+        end
+
+        def config
+          @config || Trifle::Docs.default
+        end
+
+        def perform
+          config.harvester.search_for(query: query, scope: scope)
+        end
+      end
+    end
+  end
+end

data/lib/trifle/docs/version.rb

@@ -2,6 +2,6 @@
 
 module Trifle
   module Docs
-    VERSION = '0.3.0'
+    VERSION = '0.5.0'
   end
 end

data/lib/trifle/docs.rb

@@ -2,10 +2,14 @@
 
 require_relative 'docs/configuration'
 require_relative 'docs/helper/tree'
+require_relative 'docs/helper/markdown_layout'
+require_relative 'docs/helper/ai_detection'
 require_relative 'docs/harvester'
 require_relative 'docs/harvester/file'
 require_relative 'docs/harvester/markdown'
+require_relative 'docs/operations/search'
 require_relative 'docs/operations/content'
+require_relative 'docs/operations/raw_content'
 require_relative 'docs/operations/collection'
 require_relative 'docs/operations/meta'
 require_relative 'docs/operations/sitemap'
@@ -28,6 +32,12 @@ module Trifle
       default
     end
 
+    def self.search(query:, scope: nil, config: nil)
+      Trifle::Docs::Operations::Search.new(
+        query: query, scope: scope, config: config
+      ).perform
+    end
+
     def self.content(url:, config: nil)
       Trifle::Docs::Operations::Content.new(
         url: url, config: config
@@ -40,6 +50,12 @@ module Trifle
       ).perform
     end
 
+    def self.raw_content(url:, config: nil)
+      Trifle::Docs::Operations::RawContent.new(
+        url: url, config: config
+      ).perform
+    end
+
     def self.collection(url:, config: nil)
       Trifle::Docs::Operations::Collection.new(
         url: url, config: config

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: trifle-docs
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Jozef Vaclavik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-09-18 00:00:00.000000000 Z
+date: 2025-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -200,10 +200,14 @@ files:
 - lib/trifle/docs/harvester.rb
 - lib/trifle/docs/harvester/file.rb
 - lib/trifle/docs/harvester/markdown.rb
+- lib/trifle/docs/helper/ai_detection.rb
+- lib/trifle/docs/helper/markdown_layout.rb
 - lib/trifle/docs/helper/tree.rb
 - lib/trifle/docs/operations/collection.rb
 - lib/trifle/docs/operations/content.rb
 - lib/trifle/docs/operations/meta.rb
+- lib/trifle/docs/operations/raw_content.rb
+- lib/trifle/docs/operations/search.rb
 - lib/trifle/docs/operations/sitemap.rb
 - lib/trifle/docs/version.rb
 - sig/trifle/docs.rbs