llms-txt-ruby 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: daed7a0ad2c84969d85309328b3bbb773a43b20f70f8b9f610aa8bdcbe0a0510
-  data.tar.gz: '0795721bd23fe13fdcfeac9745ca7bbe0ce49a1a56ddc62a07fba49a047de65d'
+  metadata.gz: 59e6ef21e4d8e7cfad82d91a1d0c6eede9528850622efef8738b718bbbe0ea43
+  data.tar.gz: 0247ceaaed63bea7ba5d86753cd77e19e032ec31561ab6fbfe35874d875cbda6
 SHA512:
-  metadata.gz: a8e4943867227ae1f031fedce39c7b30e46180295af86a99cda9f40404023127355ee23130a769d7bafe326c2dcce0aa9be694d9b11cdb0ea436305514443047
-  data.tar.gz: 2943403f4a90f1dfab03ec1e106ceeec7e5534ed61b7cf1ae26b16461ea24642b52a413552eb0cd508a810b59902541ee922ef3eacbbacab0b547da9b88a8e47
+  metadata.gz: 13bcd98389bbfd2193e1d21b3b5372bd1656145aec325a0f1456ce95cb699106008c35914cdab9c6ed4e7cefa4c955791347179e003def2a8569ab267c387abc
+  data.tar.gz: 4115db81602c0137224f7e5c8b9c536b5684776c3398e04893d6f1e0b2a327af71918d34c661d83eae9e773654ef282947fcd231af0a3c4cfe8263f085455381

data/.gitignore

@@ -9,6 +9,7 @@
 /test/tmp/
 /test/version_tmp/
 /tmp/
+mise.toml
 
 # Used by dotenv library to load environment variables.
 .env

data/.ruby-version

@@ -1 +1 @@
-3.4.5
+3.4.6

data/CHANGELOG.md

@@ -0,0 +1,15 @@
+# Changelog
+
+## 0.1.1 (2025-10-07)
+- [Change] Updated repository metadata to use `master` branch instead of `main`.
+
+## 0.1.0 (2025-10-07)
+- [Feature] Generate `llms.txt` files from markdown documentation.
+- [Feature] Transform individual markdown files to be AI-friendly.
+- [Feature] Bulk transformation of entire documentation directories.
+- [Feature] CLI with commands: `generate`, `transform`, `bulk-transform`, `parse`, `validate`.
+- [Feature] Configuration file support (`llms-txt.yml`).
+- [Feature] Automatic link expansion from relative to absolute URLs.
+- [Feature] File prioritization (README first, then guides, APIs, etc.).
+- [Feature] Exclusion patterns for bulk transformations.
+- [Feature] Ruby API for programmatic usage.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    llms-txt-ruby (0.1.0)
+    llms-txt-ruby (0.1.1)
       zeitwerk (~> 2.6)
 
 GEM

data/README.md

@@ -213,6 +213,169 @@ wiki/
     └── internal.md      ← Excluded, no .llm.md version
 ```
 
+## Serving LLM-Friendly Documentation
+
+After using `bulk-transform` to create `.llm.md` versions of your documentation, you can configure your web server to automatically serve these LLM-optimized versions to AI bots while showing the original versions to human visitors.
+
+### How It Works
+
+The strategy is simple:
+
+1. **Detect AI bots** by their User-Agent strings
+2. **Serve `.llm.md` files** to detected AI bots
+3. **Serve original `.md` files** to human visitors
+4. **Automatic selection** - no manual switching needed
+
+### Apache Configuration
+
+Add this to your `.htaccess` file:
+
+```apache
+# Detect LLM bots by User-Agent
+SetEnvIf User-Agent "(?i)(openai|anthropic|claude|gpt|chatgpt|bard|gemini|copilot)" IS_LLM_BOT
+SetEnvIf User-Agent "(?i)(perplexity|character\.ai|you\.com|poe\.com|huggingface|replicate)" IS_LLM_BOT
+SetEnvIf User-Agent "(?i)(langchain|llamaindex|semantic|embedding|vector|rag)" IS_LLM_BOT
+SetEnvIf User-Agent "(?i)(ollama|mistral|cohere|together|fireworks|groq)" IS_LLM_BOT
+
+# Serve .md files as text/plain
+<FilesMatch "\.md$">
+  Header set Content-Type "text/plain; charset=utf-8"
+  ForceType text/plain
+</FilesMatch>
+
+# Enable rewrite engine
+RewriteEngine On
+
+# For LLM bots: rewrite requests to serve .llm.md versions
+RewriteCond %{ENV:IS_LLM_BOT} !^$
+RewriteCond %{REQUEST_URI} ^/docs/.*\.md$ [NC]
+RewriteCond %{REQUEST_URI} !\.llm\.md$ [NC]
+RewriteCond %{DOCUMENT_ROOT}%{REQUEST_URI} -f
+RewriteRule ^(.*)\.md$ $1.llm.md [L]
+
+# For LLM bots: handle clean URLs by appending .llm.md
+RewriteCond %{ENV:IS_LLM_BOT} !^$
+RewriteCond %{REQUEST_URI} ^/docs/ [NC]
+RewriteCond %{REQUEST_URI} !\.md$ [NC]
+RewriteCond %{DOCUMENT_ROOT}%{REQUEST_URI}.llm.md -f
+RewriteRule ^(.*)$ $1.llm.md [L]
+
+# For regular users: serve original .md files or clean URLs as usual
+# (add your normal URL handling rules here)
+```
+
+### Nginx Configuration
+
+Add this to your nginx server block:
+
+```nginx
+# Map to detect LLM bots
+map $http_user_agent $is_llm_bot {
+    default 0;
+    "~*(?i)(openai|anthropic|claude|gpt|chatgpt|bard|gemini|copilot)" 1;
+    "~*(?i)(perplexity|character\.ai|you\.com|poe\.com|huggingface|replicate)" 1;
+    "~*(?i)(langchain|llamaindex|semantic|embedding|vector|rag)" 1;
+    "~*(?i)(ollama|mistral|cohere|together|fireworks|groq)" 1;
+}
+
+server {
+    # ... your server configuration ...
+
+    # Serve .md files as text/plain
+    location ~ \.md$ {
+        default_type text/plain;
+        charset utf-8;
+    }
+
+    # For LLM bots requesting .md files, serve .llm.md version
+    location ~ ^/docs/(.*)\.md$ {
+        if ($is_llm_bot) {
+            rewrite ^(.*)\.md$ $1.llm.md last;
+        }
+        try_files $uri $uri/ =404;
+    }
+
+    # For LLM bots requesting clean URLs, serve .llm.md version
+    location ~ ^/docs/ {
+        if ($is_llm_bot) {
+            try_files $uri.llm.md $uri $uri/ =404;
+        }
+        try_files $uri $uri.md $uri/ =404;
+    }
+}
+```
+
+### Cloudflare Workers
+
+For serverless deployments, use Cloudflare Workers:
+
+```javascript
+export default {
+  async fetch(request) {
+    const url = new URL(request.url);
+    const userAgent = request.headers.get('user-agent') || '';
+
+    // Detect LLM bots
+    const llmBotPatterns = [
+      /openai|anthropic|claude|gpt|chatgpt|bard|gemini|copilot/i,
+      /perplexity|character\.ai|you\.com|poe\.com|huggingface|replicate/i,
+      /langchain|llamaindex|semantic|embedding|vector|rag/i,
+      /ollama|mistral|cohere|together|fireworks|groq/i
+    ];
+
+    const isLLMBot = llmBotPatterns.some(pattern => pattern.test(userAgent));
+
+    // If LLM bot and requesting docs
+    if (isLLMBot && url.pathname.startsWith('/docs/')) {
+      // Try to serve .llm.md version
+      const llmPath = url.pathname.replace(/\.md$/, '.llm.md');
+      if (!url.pathname.endsWith('.llm.md')) {
+        url.pathname = llmPath;
+      }
+    }
+
+    return fetch(url);
+  }
+}
+```
+
+### Custom Suffix
+
+If you used a different suffix with the `bulk-transform` command (e.g., `--suffix .ai`), update your web server rules accordingly.
+
+**Apache:**
+```apache
+RewriteRule ^(.*)\.md$ $1.ai.md [L]
+```
+
+**Nginx:**
+```nginx
+rewrite ^(.*)\.md$ $1.ai.md last;
+```
+
+**Cloudflare Workers:**
+```javascript
+const llmPath = url.pathname.replace(/\.md$/, '.ai.md');
+```
+
+### Example Setup
+
+```yaml
+# llms-txt.yml
+docs: ./docs
+base_url: https://myproject.io
+suffix: .llm
+convert_urls: true
+```
+
+```bash
+# Generate LLM-friendly versions
+llms-txt bulk-transform --config llms-txt.yml
+
+# Deploy both original and .llm.md files to your web server
+# The server will automatically serve the right version to each visitor
+```
+
 ## Ruby API
 
 ### Basic Usage

data/lib/llms_txt/version.rb

@@ -2,5 +2,5 @@
 
 module LlmsTxt
   # Current version of the LlmsTxt gem
-  VERSION = '0.1.0'
+  VERSION = '0.1.1'
 end

data/llms-txt-ruby.gemspec

@@ -8,22 +8,24 @@ Gem::Specification.new do |spec|
   spec.authors = ['Maciej Mensfeld']
   spec.email = %w[maciej@mensfeld.pl]
 
-  spec.summary = 'Simple tool for generating llms.txt files from existing markdown documentation'
+  spec.summary = 'Generate llms.txt files and transform markdown documentation to be AI-friendly'
   spec.description = <<~DESC
-    A simple Ruby gem for generating llms.txt files from existing markdown documentation.
-    Transform your docs to be AI-friendly with two core functions: generate llms.txt from
-    documentation directories and transform individual markdown files by expanding relative
-    links and converting URLs.
+    A Ruby tool for transforming existing markdown documentation into AI-friendly formats
+    following the llms.txt standard. Features include: generating llms.txt files from
+    documentation directories with automatic file prioritization, transforming individual
+    markdown files by expanding relative links to absolute URLs, and bulk transforming entire
+    documentation trees with customizable exclusion patterns. Provides both CLI and Ruby API
+    with configuration file support.
   DESC
 
   spec.homepage = 'https://github.com/mensfeld/llms-txt-ruby'
   spec.license = 'MIT'
-  spec.required_ruby_version = '>= 3.1.0'
+  spec.required_ruby_version = '>= 3.2'
 
   spec.metadata['allowed_push_host'] = 'https://rubygems.org'
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = 'https://github.com/mensfeld/llms-txt-ruby'
-  spec.metadata['changelog_uri'] = 'https://github.com/mensfeld/llms-txt-ruby/blob/main/CHANGELOG.md'
+  spec.metadata['changelog_uri'] = 'https://github.com/mensfeld/llms-txt-ruby/blob/master/CHANGELOG.md'
   spec.metadata['documentation_uri'] = 'https://github.com/mensfeld/llms-txt-ruby'
   spec.metadata['rubygems_mfa_required'] = 'true'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llms-txt-ruby
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -94,10 +94,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.21'
 description: |
-  A simple Ruby gem for generating llms.txt files from existing markdown documentation.
-  Transform your docs to be AI-friendly with two core functions: generate llms.txt from
-  documentation directories and transform individual markdown files by expanding relative
-  links and converting URLs.
+  A Ruby tool for transforming existing markdown documentation into AI-friendly formats
+  following the llms.txt standard. Features include: generating llms.txt files from
+  documentation directories with automatic file prioritization, transforming individual
+  markdown files by expanding relative links to absolute URLs, and bulk transforming entire
+  documentation trees with customizable exclusion patterns. Provides both CLI and Ruby API
+  with configuration file support.
 email:
 - maciej@mensfeld.pl
 executables:
@@ -110,6 +112,7 @@ files:
 - ".gitignore"
 - ".rubocop.yml"
 - ".ruby-version"
+- CHANGELOG.md
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -127,7 +130,6 @@ files:
 - lib/llms_txt/version.rb
 - llms-txt-ruby.gemspec
 - llms-txt.yml.example
-- mise.toml
 - renovate.json
 homepage: https://github.com/mensfeld/llms-txt-ruby
 licenses:
@@ -136,7 +138,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/mensfeld/llms-txt-ruby
   source_code_uri: https://github.com/mensfeld/llms-txt-ruby
-  changelog_uri: https://github.com/mensfeld/llms-txt-ruby/blob/main/CHANGELOG.md
+  changelog_uri: https://github.com/mensfeld/llms-txt-ruby/blob/master/CHANGELOG.md
   documentation_uri: https://github.com/mensfeld/llms-txt-ruby
   rubygems_mfa_required: 'true'
 rdoc_options: []
@@ -146,7 +148,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -155,5 +157,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 3.6.9
 specification_version: 4
-summary: Simple tool for generating llms.txt files from existing markdown documentation
+summary: Generate llms.txt files and transform markdown documentation to be AI-friendly
 test_files: []

data/mise.toml

@@ -1,2 +0,0 @@
-[tools]
-ruby = "3.4.5"