llms-txt-ruby 0.0.0 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 57be54434134c87cc4939264861e7e64f1b721aeef9e52ebe11080b3f1ab08d1
-  data.tar.gz: d413c3a1902cd259f9af274aefbcaf65df9714ba7e6ae958de2afc30101b4778
+  metadata.gz: daed7a0ad2c84969d85309328b3bbb773a43b20f70f8b9f610aa8bdcbe0a0510
+  data.tar.gz: '0795721bd23fe13fdcfeac9745ca7bbe0ce49a1a56ddc62a07fba49a047de65d'
 SHA512:
-  metadata.gz: 43ab2bcc770fb9f0486363b4904f006d6b462a6adfae1b9e7d05d8a1aed5c2b4356f02ca2737ca4c6e1d0a333f132d621d5cbb1033a28afcff8c8bafcd690b4a
-  data.tar.gz: ba760b69ca401f1b3cf0e5ec14a9e76bef92e86c5804a040b03b203aa6af1e39361cbe8fb5e8fbb84ecbf48d557cb3ccd49e194f0e7e2eaeb872f0f7f07a171b
+  metadata.gz: a8e4943867227ae1f031fedce39c7b30e46180295af86a99cda9f40404023127355ee23130a769d7bafe326c2dcce0aa9be694d9b11cdb0ea436305514443047
+  data.tar.gz: 2943403f4a90f1dfab03ec1e106ceeec7e5534ed61b7cf1ae26b16461ea24642b52a413552eb0cd508a810b59902541ee922ef3eacbbacab0b547da9b88a8e47

data/.github/workflows/ci.yml

@@ -0,0 +1,71 @@
+name: CI
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    branches: [ master ]
+  push:
+    branches: [ master ]
+  schedule:
+    - cron: '0 1 * * *'
+
+permissions:
+  contents: read
+
+jobs:
+  specs:
+    timeout-minutes: 15
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - '3.4'
+          - '3.3'
+          - '3.2'
+        include:
+          - ruby: '3.4'
+            coverage: 'true'
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+        with:
+          fetch-depth: 0
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+          bundler: 'latest'
+
+      - name: Install latest bundler
+        run: |
+          gem install bundler --no-document
+          bundle config set without 'tools benchmarks docs'
+
+      - name: Bundle install
+        run: bundle install --jobs 4 --retry 3
+
+      - name: Run all tests
+        env:
+          GITHUB_COVERAGE: ${{ matrix.coverage }}
+        run: bundle exec rspec
+
+
+  ci-success:
+    name: CI Success
+    runs-on: ubuntu-latest
+    if: always()
+    needs:
+      - specs
+    steps:
+      - name: Check all jobs passed
+        if: |
+          contains(needs.*.result, 'failure') ||
+          contains(needs.*.result, 'cancelled') ||
+          contains(needs.*.result, 'skipped')
+        run: exit 1
+      - run: echo "All CI checks passed!"

data/.github/workflows/push.yml

@@ -0,0 +1,35 @@
+name: Push Gem
+
+on:
+  push:
+    tags:
+      - v*
+
+permissions:
+  contents: read
+
+jobs:
+  push:
+    if: github.repository_owner == 'mensfeld'
+    runs-on: ubuntu-latest
+    environment: deployment
+
+    permissions:
+      contents: write
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          fetch-depth: 0
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@0481980f17b760ef6bca5e8c55809102a0af1e5a # v1.263.0
+        with:
+          bundler-cache: false
+
+      - name: Bundle install
+        run: |
+          bundle install --jobs 4 --retry 3
+
+      - uses: rubygems/release-gem@a25424ba2ba8b387abc8ef40807c2c85b96cbe32 # v1.1.1

data/.gitignore

@@ -11,7 +11,8 @@
 /tmp/
 
 # Used by dotenv library to load environment variables.
-# .env
+.env
+.env.*
 
 # Ignore Byebug command history file.
 .byebug_history
@@ -54,3 +55,11 @@ build-iPhoneSimulator/
 
 # Used by RuboCop. Remote config files pulled in from inherit_from directive.
 # .rubocop-https?--*
+
+# Project-specific generated files
+llms.txt
+*-output.txt
+
+# Config files that might contain sensitive data
+llms-txt.yml
+.llms-txt.yml

data/.rubocop.yml

@@ -0,0 +1,27 @@
+AllCops:
+  TargetRubyVersion: 3.1
+  NewCops: enable
+
+Style/Documentation:
+  Enabled: false
+
+Style/StringLiterals:
+  EnforcedStyle: single_quotes
+
+Layout/LineLength:
+  Max: 120
+
+Metrics/ClassLength:
+  Max: 150
+
+Metrics/MethodLength:
+  Max: 20
+
+Metrics/AbcSize:
+  Max: 20
+
+Metrics/CyclomaticComplexity:
+  Max: 10
+
+Style/FrozenStringLiteralComment:
+  Enabled: true

data/.ruby-version

@@ -0,0 +1 @@
+3.4.5

data/Gemfile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gemspec
+
+group :development do
+  gem 'pry'
+  gem 'pry-byebug'
+end

data/Gemfile.lock

@@ -0,0 +1,88 @@
+PATH
+  remote: .
+  specs:
+    llms-txt-ruby (0.1.0)
+      zeitwerk (~> 2.6)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.3)
+    byebug (12.0.0)
+    coderay (1.1.3)
+    diff-lcs (1.6.2)
+    docile (1.4.1)
+    json (2.13.2)
+    language_server-protocol (3.17.0.5)
+    lint_roller (1.1.0)
+    method_source (1.1.0)
+    parallel (1.27.0)
+    parser (3.3.9.0)
+      ast (~> 2.4.1)
+      racc
+    prism (1.4.0)
+    pry (0.15.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.11.0)
+      byebug (~> 12.0)
+      pry (>= 0.13, < 0.16)
+    racc (1.8.1)
+    rainbow (3.1.1)
+    rake (13.3.0)
+    regexp_parser (2.11.2)
+    rspec (3.13.1)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.5)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.5)
+    rubocop (1.80.0)
+      json (~> 2.3)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.46.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.46.0)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
+    ruby-progressbar (1.13.0)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.13.2)
+    simplecov_json_formatter (0.1.4)
+    unicode-display_width (3.1.5)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
+    zeitwerk (2.7.3)
+
+PLATFORMS
+  ruby
+  x86_64-linux
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  llms-txt-ruby!
+  pry
+  pry-byebug
+  rake (~> 13.0)
+  rspec (~> 3.0)
+  rubocop (~> 1.0)
+  simplecov (~> 0.21)
+
+BUNDLED WITH
+   2.7.1

data/README.md

@@ -1,23 +1,30 @@
 # llms-txt-ruby
 
-> ⚠️ **Work in Progress** - This gem is currently under active development and not yet ready for any use.
+[![CI](https://github.com/mensfeld/llms-txt-ruby/actions/workflows/ci.yml/badge.svg)](
+  https://github.com/mensfeld/llms-txt-ruby/actions/workflows/ci.yml)
 
-A Ruby gem that automatically generates [llms.txt](https://llmstxt.org/) files for Ruby projects using AI. This gem analyzes your Ruby codebase, extracts documentation from YARD comments, README files, and gemspec metadata, then uses a Large Language Model to create a properly formatted llms.txt file following the official specification.
+A Ruby tool for generating [llms.txt](https://llmstxt.org/) files from existing markdown
+documentation. Transform your docs to be AI-friendly.
 
 ## What is llms.txt?
 
-The llms.txt file is a proposed standard for providing LLM-friendly content on websites. It offers brief background information, guidance, and links to detailed markdown files, helping Large Language Models understand and navigate your project more effectively.
+The llms.txt file is a proposed standard for providing LLM-friendly content on websites. It
+offers brief background information, guidance, and links to detailed markdown files, helping
+Large Language Models understand and navigate your project more effectively.
 
 Learn more at [llmstxt.org](https://llmstxt.org/).
 
-## Features
+## What This Tool Does
 
-- 🤖 **AI-powered generation**: Uses Claude or GPT models to create natural, comprehensive llms.txt files
-- 📚 **YARD integration**: Extracts rich documentation from YARD comments and tags
-- 🔧 **Configurable**: Supports multiple LLM providers and customizable options
-- 🖥️ **CLI + API**: Use from command line or integrate into your Ruby applications
-- 📁 **Project awareness**: Understands Ruby project structure and conventions
-- 🎯 **Spec compliant**: Generates files that strictly follow the llms.txt specification
+This library converts existing human-first documentation into LLM-friendly formats:
+
+1. **Generates llms.txt** - Transforms your existing markdown documentation into a structured
+   overview that helps LLMs understand your project's layout and find relevant information
+2. **Transforms markdown** - Converts individual markdown files from human-readable format to
+   AI-optimized format by expanding relative links to absolute URLs and normalizing link
+   structures
+3. **Bulk transforms** - Processes all markdown files in a directory recursively, creating
+   LLM-friendly versions alongside originals with customizable exclusion patterns
 
 ## Installation
 
@@ -39,38 +46,280 @@ Or install it yourself as:
 $ gem install llms-txt-ruby
 ```
 
-## Example Output
+## Quick Start
 
-Here's what a generated llms.txt file might look like:
+### Option 1: Using Config File (Recommended)
 
-```markdown
-# MyAwesomeGem
+Create a `llms-txt.yml` file in your project root:
 
-> MyAwesomeGem is a Ruby library for processing data with advanced algorithms and providing a clean API for developers.
+```yaml
+# llms-txt.yml
+docs: ./docs
+base_url: https://myproject.io
+title: My Awesome Project
+description: A Ruby library that helps developers build amazing applications
+output: llms.txt
+convert_urls: true
+verbose: false
+```
 
-This gem provides a comprehensive toolkit for data processing, featuring both synchronous and asynchronous processing capabilities. It includes built-in caching, error handling, and extensive configuration options.
+Then simply run:
 
-## Documentation
+```bash
+llms-txt generate
+```
 
-- [Getting Started Guide](docs/getting_started.md): Quick introduction and basic usage examples
-- [API Documentation](https://rubydoc.info/gems/my_awesome_gem): Complete API reference
-- [Configuration Guide](docs/configuration.md): Detailed configuration options
+### Option 2: Using CLI Only
 
-## Examples
+```bash
+# Generate from docs directory
+llms-txt generate --docs ./docs
 
-- [Basic Usage Examples](examples/basic_usage.rb): Simple examples to get started
-- [Advanced Patterns](examples/advanced_patterns.rb): Complex usage patterns and best practices
+# Transform a single file
+llms-txt transform README.md
 
-## Optional
+# Transform all markdown files in directory
+llms-txt bulk-transform --docs ./docs
 
-- [Contributing Guidelines](CONTRIBUTING.md): How to contribute to this project
-- [Changelog](CHANGELOG.md): Version history and changes
+# Use custom config file
+llms-txt generate --config my-config.yml
 ```
 
-## License
+## CLI Reference
+
+### Commands
+
+```bash
+llms-txt generate [options]       # Generate llms.txt from documentation (default)
+llms-txt transform [file]         # Transform a markdown file to be AI-friendly
+llms-txt bulk-transform [options] # Transform all markdown files in directory
+llms-txt parse [file]             # Parse existing llms.txt file
+llms-txt validate [file]          # Validate llms.txt file
+llms-txt version                  # Show version
+```
+
+### Options
+
+```bash
+-c, --config PATH        Configuration file path (default: llms-txt.yml)
+-d, --docs PATH          Path to documentation directory or file
+-o, --output PATH        Output file path
+-v, --verbose            Verbose output
+-h, --help               Show help message
+```
+
+*For advanced options like base_url, title, description, and convert_urls, use a config file.*
+
+## Configuration File
+
+The recommended way to use llms-txt is with a `llms-txt.yml` config file. This allows you to:
+
+- ✅ Store all your settings in one place
+- ✅ Version control your llms.txt configuration
+- ✅ Avoid typing long CLI commands repeatedly
+- ✅ Share configuration across team members
 
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+### Config File Options
 
----
+```yaml
+# Path to documentation directory or file
+docs: ./docs
+
+# Base URL for expanding relative links (optional)
+base_url: https://myproject.io
+
+# Project information (optional - auto-detected if not provided)
+title: My Project Name
+description: Brief description of what your project does
+
+# Output file (optional, default: llms.txt)
+output: llms.txt
+
+# Transformation options (optional)
+convert_urls: true   # Convert .html links to .md
+verbose: false       # Enable verbose output
+```
+
+The config file will be automatically found if named:
+- `llms-txt.yml`
+- `llms-txt.yaml`
+- `.llms-txt.yml`
+
+## Bulk Transformation
+
+The `bulk-transform` command processes all markdown files in a directory recursively, creating
+AI-friendly versions alongside the originals. This is perfect for transforming entire
+documentation trees.
+
+### Key Features
+
+- **Recursive processing** - Finds and transforms all `.md` files in nested directories
+- **Preserves structure** - Maintains your existing directory layout
+- **Exclusion patterns** - Skip files/directories using glob patterns
+- **Custom suffixes** - Choose how transformed files are named
+- **LLM optimizations** - Expands relative links, converts HTML URLs, etc.
+
+### Usage
+
+```bash
+# Transform all files with default settings
+llms-txt bulk-transform --docs ./wiki
+
+# Using config file (recommended for complex setups)
+llms-txt bulk-transform --config karafka-config.yml
+```
+
+### Example Config for Bulk Transformation
+
+```yaml
+# karafka-config.yml
+docs: ./wiki
+base_url: https://karafka.io
+suffix: .llm
+convert_urls: true
+excludes:
+  - "**/private/**"      # Skip private directories
+  - "**/draft-*.md"      # Skip draft files
+  - "**/old-docs/**"     # Skip legacy documentation
+```
+
+### Example Output
+
+With the config above, these files:
+```
+wiki/
+├── Home.md
+├── getting-started.md
+├── api/
+│   ├── consumers.md
+│   └── producers.md
+└── private/
+    └── internal.md
+```
+
+Become:
+```
+wiki/
+├── Home.md
+├── Home.llm.md          ← AI-friendly version
+├── getting-started.md
+├── getting-started.llm.md
+├── api/
+│   ├── consumers.md
+│   ├── consumers.llm.md
+│   ├── producers.md
+│   └── producers.llm.md
+└── private/
+    └── internal.md      ← Excluded, no .llm.md version
+```
+
+## Ruby API
+
+### Basic Usage
+
+```ruby
+require 'llms_txt'
+
+# Option 1: Using config file (recommended)
+content = LlmsTxt.generate_from_docs(config_file: 'llms-txt.yml')
+
+# Option 2: Direct options (overrides config)
+content = LlmsTxt.generate_from_docs('./docs',
+  base_url: 'https://myproject.io',
+  title: 'My Project',
+  description: 'A great project'
+)
+
+# Option 3: Mix config file with overrides
+content = LlmsTxt.generate_from_docs('./docs',
+  config_file: 'my-config.yml',
+  title: 'Override Title'  # This overrides config file title
+)
+
+# Transform markdown with config
+transformed = LlmsTxt.transform_markdown('README.md',
+  config_file: 'llms-txt.yml'
+)
+
+# Transform with direct options
+transformed = LlmsTxt.transform_markdown('README.md',
+  base_url: 'https://myproject.io',
+  convert_urls: true
+)
+
+# Bulk transform all files in directory
+transformed_files = LlmsTxt.bulk_transform('./wiki',
+  base_url: 'https://karafka.io',
+  suffix: '.llm',
+  excludes: ['**/private/**', '**/draft-*.md']
+)
+puts "Transformed #{transformed_files.size} files"
+
+# Bulk transform with config file
+transformed_files = LlmsTxt.bulk_transform('./wiki',
+  config_file: 'karafka-config.yml'
+)
+
+# Parse and validate (unchanged)
+parsed = LlmsTxt.parse('llms.txt')
+puts parsed.title
+puts parsed.description
+
+valid = LlmsTxt.validate(content)
+```
+
+## How It Works
+
+### Generation Process
+
+1. **Scan for markdown files** - Finds all `.md` files in specified directory
+2. **Extract metadata** - Gets title and description from each file
+3. **Prioritize docs** - Orders by importance (README first, then guides, APIs, etc.)
+4. **Build llms.txt** - Creates properly formatted output with links and descriptions
+
+### Transformation Process
+
+1. **Expand relative links** - Convert `./docs/api.md` to `https://myproject.io/docs/api.md`
+2. **Convert URLs** - Change `.html` links to `.md` for better AI understanding
+3. **Preserve content** - No content modification, just link processing
+
+### File Prioritization
+
+When generating llms.txt, files are automatically prioritized:
+
+1. **README files** - Always listed first
+2. **Getting Started guides** - Quick start documentation
+3. **Guides and tutorials** - Step-by-step content
+4. **API references** - Technical documentation
+5. **Other files** - Everything else
+
+## Example Output
+
+Given a `docs/` directory with:
+- `README.md`
+- `getting-started.md`
+- `api-reference.md`
+
+Running `llms-txt generate --docs ./docs --base-url https://myproject.io` creates:
+
+```markdown
+# My Project
+
+> This is a Ruby library that helps developers build amazing applications with a clean, simple API.
+
+## Documentation
+
+- [README](https://myproject.io/README.md): Complete overview and installation instructions
+- [Getting Started](https://myproject.io/getting-started.md): Quick start guide with examples
+- [API Reference](https://myproject.io/api-reference.md): Detailed API documentation and method
+  signatures
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/mensfeld/llms-txt-ruby.
+
+## License
 
-Made with ❤️ for the Ruby community
+The gem is available as open source under the terms of the
+[MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]